2024-11-17 08:14:02,157 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-17 08:14:02,169 main DEBUG Took 0.009513 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-17 08:14:02,169 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-17 08:14:02,169 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-17 08:14:02,170 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-17 08:14:02,172 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 08:14:02,181 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-17 08:14:02,197 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 08:14:02,199 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 08:14:02,200 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 08:14:02,201 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 08:14:02,201 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 08:14:02,202 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 08:14:02,204 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 08:14:02,204 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 08:14:02,205 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 08:14:02,205 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 08:14:02,206 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 08:14:02,207 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 08:14:02,208 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 08:14:02,208 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 08:14:02,209 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 08:14:02,209 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 08:14:02,210 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 08:14:02,210 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 08:14:02,211 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 08:14:02,212 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 08:14:02,212 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 08:14:02,213 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 08:14:02,213 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 08:14:02,214 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 08:14:02,214 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 08:14:02,215 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-17 08:14:02,217 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 08:14:02,218 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-17 08:14:02,221 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-17 08:14:02,221 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-17 08:14:02,223 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-17 08:14:02,224 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-17 08:14:02,234 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-17 08:14:02,238 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-17 08:14:02,240 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-17 08:14:02,241 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-17 08:14:02,241 main DEBUG createAppenders(={Console}) 2024-11-17 08:14:02,242 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-17 08:14:02,242 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-17 08:14:02,243 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-17 08:14:02,244 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-17 08:14:02,244 main DEBUG OutputStream closed 2024-11-17 08:14:02,244 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-17 08:14:02,245 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-17 08:14:02,245 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-17 08:14:02,327 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-17 08:14:02,329 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-17 08:14:02,330 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-17 08:14:02,331 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-17 08:14:02,332 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-17 08:14:02,332 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-17 08:14:02,333 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-17 08:14:02,333 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-17 08:14:02,333 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-17 08:14:02,334 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-17 08:14:02,334 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-17 08:14:02,334 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-17 08:14:02,335 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-17 08:14:02,335 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-17 08:14:02,335 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-17 08:14:02,336 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-17 08:14:02,336 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-17 08:14:02,337 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-17 08:14:02,339 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-17 08:14:02,339 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-17 08:14:02,339 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-17 08:14:02,340 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-17T08:14:02,558 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5720963-478b-ee25-58c5-86f892de1407 2024-11-17 08:14:02,561 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-17 08:14:02,562 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-17T08:14:02,569 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-17T08:14:02,594 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-17T08:14:02,613 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5720963-478b-ee25-58c5-86f892de1407/cluster_aa2b9c1b-2c01-fdb4-e68c-d41e93bcbc98, deleteOnExit=true 2024-11-17T08:14:02,613 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-17T08:14:02,615 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5720963-478b-ee25-58c5-86f892de1407/test.cache.data in system properties and HBase conf 2024-11-17T08:14:02,616 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5720963-478b-ee25-58c5-86f892de1407/hadoop.tmp.dir in system properties and HBase conf 2024-11-17T08:14:02,616 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5720963-478b-ee25-58c5-86f892de1407/hadoop.log.dir in system properties and HBase conf 2024-11-17T08:14:02,617 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5720963-478b-ee25-58c5-86f892de1407/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-17T08:14:02,618 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5720963-478b-ee25-58c5-86f892de1407/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-17T08:14:02,619 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-17T08:14:02,733 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-17T08:14:02,831 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T08:14:02,835 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5720963-478b-ee25-58c5-86f892de1407/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-17T08:14:02,836 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5720963-478b-ee25-58c5-86f892de1407/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-17T08:14:02,836 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5720963-478b-ee25-58c5-86f892de1407/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-17T08:14:02,836 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5720963-478b-ee25-58c5-86f892de1407/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-17T08:14:02,837 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5720963-478b-ee25-58c5-86f892de1407/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-17T08:14:02,837 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5720963-478b-ee25-58c5-86f892de1407/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-17T08:14:02,838 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5720963-478b-ee25-58c5-86f892de1407/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-17T08:14:02,838 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5720963-478b-ee25-58c5-86f892de1407/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-17T08:14:02,838 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5720963-478b-ee25-58c5-86f892de1407/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-17T08:14:02,839 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5720963-478b-ee25-58c5-86f892de1407/nfs.dump.dir in system properties and HBase conf 2024-11-17T08:14:02,839 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5720963-478b-ee25-58c5-86f892de1407/java.io.tmpdir in system properties and HBase conf 2024-11-17T08:14:02,839 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5720963-478b-ee25-58c5-86f892de1407/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-17T08:14:02,840 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5720963-478b-ee25-58c5-86f892de1407/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-17T08:14:02,840 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5720963-478b-ee25-58c5-86f892de1407/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-17T08:14:03,914 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-17T08:14:03,987 INFO [Time-limited test {}] log.Log(170): Logging initialized @2475ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-17T08:14:04,054 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T08:14:04,111 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-17T08:14:04,130 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-17T08:14:04,130 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-17T08:14:04,132 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-17T08:14:04,142 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T08:14:04,145 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5720963-478b-ee25-58c5-86f892de1407/hadoop.log.dir/,AVAILABLE} 2024-11-17T08:14:04,146 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-17T08:14:04,308 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5720963-478b-ee25-58c5-86f892de1407/java.io.tmpdir/jetty-localhost-45601-hadoop-hdfs-3_4_1-tests_jar-_-any-1079550556993038858/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-17T08:14:04,315 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:45601} 2024-11-17T08:14:04,316 INFO [Time-limited test {}] server.Server(415): Started @2805ms 2024-11-17T08:14:04,864 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T08:14:04,872 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-17T08:14:04,874 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-17T08:14:04,874 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-17T08:14:04,874 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-17T08:14:04,875 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5720963-478b-ee25-58c5-86f892de1407/hadoop.log.dir/,AVAILABLE} 2024-11-17T08:14:04,875 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-17T08:14:04,975 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5720963-478b-ee25-58c5-86f892de1407/java.io.tmpdir/jetty-localhost-40585-hadoop-hdfs-3_4_1-tests_jar-_-any-9720405828368950824/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-17T08:14:04,976 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:40585} 2024-11-17T08:14:04,976 INFO [Time-limited test {}] server.Server(415): Started @3465ms 2024-11-17T08:14:05,024 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-17T08:14:05,983 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5720963-478b-ee25-58c5-86f892de1407/cluster_aa2b9c1b-2c01-fdb4-e68c-d41e93bcbc98/data/data2/current/BP-2116440293-172.17.0.3-1731831243397/current, will proceed with Du for space computation calculation, 2024-11-17T08:14:05,983 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5720963-478b-ee25-58c5-86f892de1407/cluster_aa2b9c1b-2c01-fdb4-e68c-d41e93bcbc98/data/data1/current/BP-2116440293-172.17.0.3-1731831243397/current, will proceed with Du for space computation calculation, 2024-11-17T08:14:06,013 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-17T08:14:06,057 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x58ca552261b16a43 with lease ID 0x1e38c7cd21639d8c: Processing first storage report for DS-7eaf6057-7747-4a5f-8ff7-5300ff40dabc from datanode DatanodeRegistration(127.0.0.1:40497, datanodeUuid=bc39c235-fe1b-4242-b2b6-1d3bc0ce3f3c, infoPort=42919, infoSecurePort=0, ipcPort=46851, storageInfo=lv=-57;cid=testClusterID;nsid=1307733217;c=1731831243397) 2024-11-17T08:14:06,059 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x58ca552261b16a43 with lease ID 0x1e38c7cd21639d8c: from storage DS-7eaf6057-7747-4a5f-8ff7-5300ff40dabc node DatanodeRegistration(127.0.0.1:40497, datanodeUuid=bc39c235-fe1b-4242-b2b6-1d3bc0ce3f3c, infoPort=42919, infoSecurePort=0, ipcPort=46851, storageInfo=lv=-57;cid=testClusterID;nsid=1307733217;c=1731831243397), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-17T08:14:06,059 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x58ca552261b16a43 with lease ID 0x1e38c7cd21639d8c: Processing first storage report for DS-dab9604c-026f-4a5a-ae78-54ec5fcf06ec from datanode DatanodeRegistration(127.0.0.1:40497, datanodeUuid=bc39c235-fe1b-4242-b2b6-1d3bc0ce3f3c, infoPort=42919, infoSecurePort=0, ipcPort=46851, storageInfo=lv=-57;cid=testClusterID;nsid=1307733217;c=1731831243397) 2024-11-17T08:14:06,059 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x58ca552261b16a43 with lease ID 0x1e38c7cd21639d8c: from storage DS-dab9604c-026f-4a5a-ae78-54ec5fcf06ec node DatanodeRegistration(127.0.0.1:40497, datanodeUuid=bc39c235-fe1b-4242-b2b6-1d3bc0ce3f3c, infoPort=42919, infoSecurePort=0, ipcPort=46851, storageInfo=lv=-57;cid=testClusterID;nsid=1307733217;c=1731831243397), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-17T08:14:06,114 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5720963-478b-ee25-58c5-86f892de1407 2024-11-17T08:14:06,178 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5720963-478b-ee25-58c5-86f892de1407/cluster_aa2b9c1b-2c01-fdb4-e68c-d41e93bcbc98/zookeeper_0, clientPort=63364, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5720963-478b-ee25-58c5-86f892de1407/cluster_aa2b9c1b-2c01-fdb4-e68c-d41e93bcbc98/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5720963-478b-ee25-58c5-86f892de1407/cluster_aa2b9c1b-2c01-fdb4-e68c-d41e93bcbc98/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-17T08:14:06,187 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63364 2024-11-17T08:14:06,197 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T08:14:06,200 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T08:14:06,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741825_1001 (size=7) 2024-11-17T08:14:06,822 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85 with version=8 2024-11-17T08:14:06,823 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/hbase-staging 2024-11-17T08:14:06,905 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-17T08:14:07,141 INFO [Time-limited test {}] client.ConnectionUtils(128): master/e3d0adb87c01:0 server-side Connection retries=45 2024-11-17T08:14:07,150 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T08:14:07,150 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T08:14:07,154 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T08:14:07,154 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T08:14:07,155 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T08:14:07,280 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-17T08:14:07,330 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-17T08:14:07,337 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-17T08:14:07,340 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T08:14:07,362 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4798 (auto-detected) 2024-11-17T08:14:07,363 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-17T08:14:07,379 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:45089 2024-11-17T08:14:07,397 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45089 connecting to ZooKeeper ensemble=127.0.0.1:63364 2024-11-17T08:14:07,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:450890x0, quorum=127.0.0.1:63364, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T08:14:07,564 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45089-0x10147dba8ff0000 connected 2024-11-17T08:14:07,696 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T08:14:07,700 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T08:14:07,711 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45089-0x10147dba8ff0000, quorum=127.0.0.1:63364, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T08:14:07,714 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85, hbase.cluster.distributed=false 2024-11-17T08:14:07,735 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45089-0x10147dba8ff0000, quorum=127.0.0.1:63364, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T08:14:07,740 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45089 2024-11-17T08:14:07,740 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45089 2024-11-17T08:14:07,741 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45089 2024-11-17T08:14:07,741 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45089 2024-11-17T08:14:07,741 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45089 2024-11-17T08:14:07,837 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e3d0adb87c01:0 server-side Connection retries=45 2024-11-17T08:14:07,838 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T08:14:07,838 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T08:14:07,839 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T08:14:07,839 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T08:14:07,839 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T08:14:07,842 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-17T08:14:07,845 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T08:14:07,846 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40631 2024-11-17T08:14:07,847 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40631 connecting to ZooKeeper ensemble=127.0.0.1:63364 2024-11-17T08:14:07,849 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T08:14:07,851 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T08:14:07,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:406310x0, quorum=127.0.0.1:63364, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T08:14:07,870 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40631-0x10147dba8ff0001 connected 2024-11-17T08:14:07,870 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40631-0x10147dba8ff0001, quorum=127.0.0.1:63364, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T08:14:07,874 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-17T08:14:07,881 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-17T08:14:07,884 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40631-0x10147dba8ff0001, quorum=127.0.0.1:63364, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-17T08:14:07,889 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40631-0x10147dba8ff0001, quorum=127.0.0.1:63364, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T08:14:07,890 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40631 2024-11-17T08:14:07,890 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40631 2024-11-17T08:14:07,890 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40631 2024-11-17T08:14:07,891 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40631 2024-11-17T08:14:07,891 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40631 2024-11-17T08:14:07,905 DEBUG [M:0;e3d0adb87c01:45089 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;e3d0adb87c01:45089 2024-11-17T08:14:07,906 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/e3d0adb87c01,45089,1731831246985 2024-11-17T08:14:07,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45089-0x10147dba8ff0000, quorum=127.0.0.1:63364, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T08:14:07,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40631-0x10147dba8ff0001, quorum=127.0.0.1:63364, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T08:14:07,925 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45089-0x10147dba8ff0000, quorum=127.0.0.1:63364, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/e3d0adb87c01,45089,1731831246985 2024-11-17T08:14:07,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45089-0x10147dba8ff0000, quorum=127.0.0.1:63364, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T08:14:07,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40631-0x10147dba8ff0001, quorum=127.0.0.1:63364, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-17T08:14:07,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40631-0x10147dba8ff0001, quorum=127.0.0.1:63364, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T08:14:07,958 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45089-0x10147dba8ff0000, quorum=127.0.0.1:63364, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-17T08:14:07,959 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/e3d0adb87c01,45089,1731831246985 from backup master directory 2024-11-17T08:14:07,967 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45089-0x10147dba8ff0000, quorum=127.0.0.1:63364, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/e3d0adb87c01,45089,1731831246985 2024-11-17T08:14:07,967 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40631-0x10147dba8ff0001, quorum=127.0.0.1:63364, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T08:14:07,967 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45089-0x10147dba8ff0000, quorum=127.0.0.1:63364, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T08:14:07,968 WARN [master/e3d0adb87c01:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T08:14:07,968 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=e3d0adb87c01,45089,1731831246985 2024-11-17T08:14:07,970 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-17T08:14:07,971 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-17T08:14:08,022 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/hbase.id] with ID: 15a697f6-d2a3-47b3-8935-91acf4ef5fd5 2024-11-17T08:14:08,022 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/.tmp/hbase.id 2024-11-17T08:14:08,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741826_1002 (size=42) 2024-11-17T08:14:08,436 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/.tmp/hbase.id]:[hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/hbase.id] 2024-11-17T08:14:08,483 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T08:14:08,489 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-17T08:14:08,507 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-17T08:14:08,568 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40631-0x10147dba8ff0001, quorum=127.0.0.1:63364, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T08:14:08,568 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45089-0x10147dba8ff0000, quorum=127.0.0.1:63364, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T08:14:08,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741827_1003 (size=196) 2024-11-17T08:14:09,009 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T08:14:09,011 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-17T08:14:09,025 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:09,028 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T08:14:09,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741828_1004 (size=1189) 2024-11-17T08:14:09,473 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/MasterData/data/master/store 2024-11-17T08:14:09,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741829_1005 (size=34) 2024-11-17T08:14:09,904 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-17T08:14:09,907 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T08:14:09,908 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-17T08:14:09,908 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T08:14:09,908 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T08:14:09,909 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-17T08:14:09,910 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T08:14:09,910 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T08:14:09,911 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731831249908Disabling compacts and flushes for region at 1731831249908Disabling writes for close at 1731831249909 (+1 ms)Writing region close event to WAL at 1731831249910 (+1 ms)Closed at 1731831249910 2024-11-17T08:14:09,913 WARN [master/e3d0adb87c01:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/MasterData/data/master/store/.initializing 2024-11-17T08:14:09,913 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/MasterData/WALs/e3d0adb87c01,45089,1731831246985 2024-11-17T08:14:09,922 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T08:14:09,937 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e3d0adb87c01%2C45089%2C1731831246985, suffix=, logDir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/MasterData/WALs/e3d0adb87c01,45089,1731831246985, archiveDir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/MasterData/oldWALs, maxLogs=10 2024-11-17T08:14:09,966 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/MasterData/WALs/e3d0adb87c01,45089,1731831246985/e3d0adb87c01%2C45089%2C1731831246985.1731831249942, exclude list is [], retry=0 2024-11-17T08:14:09,983 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40497,DS-7eaf6057-7747-4a5f-8ff7-5300ff40dabc,DISK] 2024-11-17T08:14:09,986 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-17T08:14:10,027 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/MasterData/WALs/e3d0adb87c01,45089,1731831246985/e3d0adb87c01%2C45089%2C1731831246985.1731831249942 2024-11-17T08:14:10,028 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42919:42919)] 2024-11-17T08:14:10,028 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-17T08:14:10,029 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T08:14:10,033 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T08:14:10,034 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T08:14:10,072 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T08:14:10,098 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-17T08:14:10,102 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:14:10,104 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T08:14:10,105 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T08:14:10,108 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-17T08:14:10,108 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:14:10,109 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T08:14:10,110 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T08:14:10,113 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-17T08:14:10,113 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:14:10,114 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T08:14:10,114 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T08:14:10,119 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-17T08:14:10,120 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:14:10,121 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T08:14:10,122 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T08:14:10,127 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-17T08:14:10,128 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-17T08:14:10,136 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T08:14:10,137 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T08:14:10,142 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-17T08:14:10,147 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T08:14:10,153 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T08:14:10,155 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60070160, jitterRate=-0.10488486289978027}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-17T08:14:10,162 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731831250047Initializing all the Stores at 1731831250049 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731831250050 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731831250051 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731831250051Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731831250051Cleaning up temporary data from old regions at 1731831250137 (+86 ms)Region opened successfully at 1731831250162 (+25 ms) 2024-11-17T08:14:10,163 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-17T08:14:10,205 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ebbac29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e3d0adb87c01/172.17.0.3:0 2024-11-17T08:14:10,237 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-17T08:14:10,247 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-17T08:14:10,247 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-17T08:14:10,250 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-17T08:14:10,252 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-17T08:14:10,256 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-17T08:14:10,256 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-17T08:14:10,280 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-17T08:14:10,290 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45089-0x10147dba8ff0000, quorum=127.0.0.1:63364, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-17T08:14:10,304 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-17T08:14:10,306 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-17T08:14:10,307 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45089-0x10147dba8ff0000, quorum=127.0.0.1:63364, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-17T08:14:10,314 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-17T08:14:10,316 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-17T08:14:10,321 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45089-0x10147dba8ff0000, quorum=127.0.0.1:63364, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-17T08:14:10,332 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-17T08:14:10,336 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45089-0x10147dba8ff0000, quorum=127.0.0.1:63364, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-17T08:14:10,367 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-17T08:14:10,386 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45089-0x10147dba8ff0000, quorum=127.0.0.1:63364, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-17T08:14:10,406 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-17T08:14:10,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40631-0x10147dba8ff0001, quorum=127.0.0.1:63364, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-17T08:14:10,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45089-0x10147dba8ff0000, quorum=127.0.0.1:63364, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-17T08:14:10,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45089-0x10147dba8ff0000, quorum=127.0.0.1:63364, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T08:14:10,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40631-0x10147dba8ff0001, quorum=127.0.0.1:63364, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T08:14:10,422 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=e3d0adb87c01,45089,1731831246985, sessionid=0x10147dba8ff0000, setting cluster-up flag (Was=false) 2024-11-17T08:14:10,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45089-0x10147dba8ff0000, quorum=127.0.0.1:63364, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T08:14:10,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40631-0x10147dba8ff0001, quorum=127.0.0.1:63364, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T08:14:10,483 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-17T08:14:10,485 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e3d0adb87c01,45089,1731831246985 2024-11-17T08:14:10,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40631-0x10147dba8ff0001, quorum=127.0.0.1:63364, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T08:14:10,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45089-0x10147dba8ff0000, quorum=127.0.0.1:63364, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T08:14:10,777 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-17T08:14:10,779 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e3d0adb87c01,45089,1731831246985 2024-11-17T08:14:10,784 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-17T08:14:10,797 INFO [RS:0;e3d0adb87c01:40631 {}] regionserver.HRegionServer(746): ClusterId : 15a697f6-d2a3-47b3-8935-91acf4ef5fd5 2024-11-17T08:14:10,800 DEBUG [RS:0;e3d0adb87c01:40631 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-17T08:14:10,848 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-17T08:14:10,858 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-17T08:14:10,860 DEBUG [RS:0;e3d0adb87c01:40631 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-17T08:14:10,860 DEBUG [RS:0;e3d0adb87c01:40631 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-17T08:14:10,865 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-17T08:14:10,870 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: e3d0adb87c01,45089,1731831246985 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-17T08:14:10,884 DEBUG [RS:0;e3d0adb87c01:40631 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-17T08:14:10,885 DEBUG [RS:0;e3d0adb87c01:40631 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ed38f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e3d0adb87c01/172.17.0.3:0 2024-11-17T08:14:10,886 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/e3d0adb87c01:0, corePoolSize=5, maxPoolSize=5 2024-11-17T08:14:10,886 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/e3d0adb87c01:0, corePoolSize=5, maxPoolSize=5 2024-11-17T08:14:10,886 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/e3d0adb87c01:0, corePoolSize=5, maxPoolSize=5 2024-11-17T08:14:10,887 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/e3d0adb87c01:0, corePoolSize=5, maxPoolSize=5 2024-11-17T08:14:10,887 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/e3d0adb87c01:0, corePoolSize=10, maxPoolSize=10 2024-11-17T08:14:10,887 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/e3d0adb87c01:0, corePoolSize=1, maxPoolSize=1 2024-11-17T08:14:10,887 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/e3d0adb87c01:0, corePoolSize=2, maxPoolSize=2 2024-11-17T08:14:10,888 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/e3d0adb87c01:0, corePoolSize=1, maxPoolSize=1 2024-11-17T08:14:10,894 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T08:14:10,895 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-17T08:14:10,897 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731831280896 2024-11-17T08:14:10,898 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-17T08:14:10,900 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-17T08:14:10,902 DEBUG [RS:0;e3d0adb87c01:40631 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;e3d0adb87c01:40631 2024-11-17T08:14:10,902 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:14:10,902 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-17T08:14:10,904 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-17T08:14:10,904 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-17T08:14:10,905 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-17T08:14:10,905 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-17T08:14:10,906 INFO [RS:0;e3d0adb87c01:40631 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-17T08:14:10,906 INFO [RS:0;e3d0adb87c01:40631 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-17T08:14:10,906 DEBUG [RS:0;e3d0adb87c01:40631 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-17T08:14:10,906 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-17T08:14:10,909 INFO [RS:0;e3d0adb87c01:40631 {}] regionserver.HRegionServer(2659): reportForDuty to master=e3d0adb87c01,45089,1731831246985 with port=40631, startcode=1731831247805 2024-11-17T08:14:10,910 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-17T08:14:10,911 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-17T08:14:10,911 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-17T08:14:10,913 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-17T08:14:10,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741831_1007 (size=1321) 2024-11-17T08:14:10,914 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-17T08:14:10,915 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/e3d0adb87c01:0:becomeActiveMaster-HFileCleaner.large.0-1731831250915,5,FailOnTimeoutGroup] 2024-11-17T08:14:10,916 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/e3d0adb87c01:0:becomeActiveMaster-HFileCleaner.small.0-1731831250915,5,FailOnTimeoutGroup] 2024-11-17T08:14:10,916 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-17T08:14:10,916 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-17T08:14:10,917 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-17T08:14:10,917 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-17T08:14:10,921 DEBUG [RS:0;e3d0adb87c01:40631 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-17T08:14:10,951 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52789, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-17T08:14:10,956 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45089 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:10,958 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45089 {}] master.ServerManager(517): Registering regionserver=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:10,970 DEBUG [RS:0;e3d0adb87c01:40631 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85 2024-11-17T08:14:10,971 DEBUG [RS:0;e3d0adb87c01:40631 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37737 2024-11-17T08:14:10,971 DEBUG [RS:0;e3d0adb87c01:40631 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-17T08:14:10,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45089-0x10147dba8ff0000, quorum=127.0.0.1:63364, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-17T08:14:10,985 DEBUG [RS:0;e3d0adb87c01:40631 {}] zookeeper.ZKUtil(111): regionserver:40631-0x10147dba8ff0001, quorum=127.0.0.1:63364, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:10,985 WARN [RS:0;e3d0adb87c01:40631 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T08:14:10,986 INFO [RS:0;e3d0adb87c01:40631 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T08:14:10,986 DEBUG [RS:0;e3d0adb87c01:40631 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/WALs/e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:10,988 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e3d0adb87c01,40631,1731831247805] 2024-11-17T08:14:11,015 INFO [RS:0;e3d0adb87c01:40631 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-17T08:14:11,032 INFO [RS:0;e3d0adb87c01:40631 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-17T08:14:11,037 INFO [RS:0;e3d0adb87c01:40631 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-17T08:14:11,037 INFO [RS:0;e3d0adb87c01:40631 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T08:14:11,038 INFO [RS:0;e3d0adb87c01:40631 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-17T08:14:11,044 INFO [RS:0;e3d0adb87c01:40631 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-17T08:14:11,046 INFO [RS:0;e3d0adb87c01:40631 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-17T08:14:11,046 DEBUG [RS:0;e3d0adb87c01:40631 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e3d0adb87c01:0, corePoolSize=1, maxPoolSize=1 2024-11-17T08:14:11,046 DEBUG [RS:0;e3d0adb87c01:40631 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e3d0adb87c01:0, corePoolSize=1, maxPoolSize=1 2024-11-17T08:14:11,046 DEBUG [RS:0;e3d0adb87c01:40631 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e3d0adb87c01:0, corePoolSize=1, maxPoolSize=1 2024-11-17T08:14:11,046 DEBUG [RS:0;e3d0adb87c01:40631 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e3d0adb87c01:0, corePoolSize=1, maxPoolSize=1 2024-11-17T08:14:11,046 DEBUG [RS:0;e3d0adb87c01:40631 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e3d0adb87c01:0, corePoolSize=1, maxPoolSize=1 2024-11-17T08:14:11,047 DEBUG [RS:0;e3d0adb87c01:40631 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e3d0adb87c01:0, corePoolSize=2, maxPoolSize=2 2024-11-17T08:14:11,047 DEBUG [RS:0;e3d0adb87c01:40631 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0, corePoolSize=1, maxPoolSize=1 2024-11-17T08:14:11,047 DEBUG [RS:0;e3d0adb87c01:40631 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e3d0adb87c01:0, corePoolSize=1, maxPoolSize=1 2024-11-17T08:14:11,047 DEBUG [RS:0;e3d0adb87c01:40631 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e3d0adb87c01:0, corePoolSize=1, maxPoolSize=1 2024-11-17T08:14:11,047 DEBUG [RS:0;e3d0adb87c01:40631 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e3d0adb87c01:0, corePoolSize=1, maxPoolSize=1 2024-11-17T08:14:11,047 DEBUG [RS:0;e3d0adb87c01:40631 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e3d0adb87c01:0, corePoolSize=1, maxPoolSize=1 2024-11-17T08:14:11,047 DEBUG [RS:0;e3d0adb87c01:40631 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e3d0adb87c01:0, corePoolSize=1, maxPoolSize=1 2024-11-17T08:14:11,047 DEBUG [RS:0;e3d0adb87c01:40631 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e3d0adb87c01:0, corePoolSize=3, maxPoolSize=3 2024-11-17T08:14:11,048 DEBUG [RS:0;e3d0adb87c01:40631 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0, corePoolSize=3, maxPoolSize=3 2024-11-17T08:14:11,048 INFO [RS:0;e3d0adb87c01:40631 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T08:14:11,049 INFO [RS:0;e3d0adb87c01:40631 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T08:14:11,049 INFO [RS:0;e3d0adb87c01:40631 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T08:14:11,049 INFO [RS:0;e3d0adb87c01:40631 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-17T08:14:11,049 INFO [RS:0;e3d0adb87c01:40631 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-17T08:14:11,049 INFO [RS:0;e3d0adb87c01:40631 {}] hbase.ChoreService(168): Chore ScheduledChore name=e3d0adb87c01,40631,1731831247805-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T08:14:11,066 INFO [RS:0;e3d0adb87c01:40631 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-17T08:14:11,067 INFO [RS:0;e3d0adb87c01:40631 {}] hbase.ChoreService(168): Chore ScheduledChore name=e3d0adb87c01,40631,1731831247805-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T08:14:11,068 INFO [RS:0;e3d0adb87c01:40631 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T08:14:11,068 INFO [RS:0;e3d0adb87c01:40631 {}] regionserver.Replication(171): e3d0adb87c01,40631,1731831247805 started 2024-11-17T08:14:11,088 INFO [RS:0;e3d0adb87c01:40631 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T08:14:11,088 INFO [RS:0;e3d0adb87c01:40631 {}] regionserver.HRegionServer(1482): Serving as e3d0adb87c01,40631,1731831247805, RpcServer on e3d0adb87c01/172.17.0.3:40631, sessionid=0x10147dba8ff0001 2024-11-17T08:14:11,089 DEBUG [RS:0;e3d0adb87c01:40631 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-17T08:14:11,089 DEBUG [RS:0;e3d0adb87c01:40631 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:11,089 DEBUG [RS:0;e3d0adb87c01:40631 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e3d0adb87c01,40631,1731831247805' 2024-11-17T08:14:11,089 DEBUG [RS:0;e3d0adb87c01:40631 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-17T08:14:11,090 DEBUG [RS:0;e3d0adb87c01:40631 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-17T08:14:11,091 DEBUG [RS:0;e3d0adb87c01:40631 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-17T08:14:11,091 DEBUG [RS:0;e3d0adb87c01:40631 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-17T08:14:11,091 DEBUG [RS:0;e3d0adb87c01:40631 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:11,091 DEBUG [RS:0;e3d0adb87c01:40631 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e3d0adb87c01,40631,1731831247805' 2024-11-17T08:14:11,091 DEBUG [RS:0;e3d0adb87c01:40631 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-17T08:14:11,092 DEBUG [RS:0;e3d0adb87c01:40631 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-17T08:14:11,093 DEBUG [RS:0;e3d0adb87c01:40631 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-17T08:14:11,093 INFO [RS:0;e3d0adb87c01:40631 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-17T08:14:11,093 INFO [RS:0;e3d0adb87c01:40631 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-17T08:14:11,201 INFO [RS:0;e3d0adb87c01:40631 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T08:14:11,207 INFO [RS:0;e3d0adb87c01:40631 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e3d0adb87c01%2C40631%2C1731831247805, suffix=, logDir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/WALs/e3d0adb87c01,40631,1731831247805, archiveDir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/oldWALs, maxLogs=32 2024-11-17T08:14:11,222 DEBUG [RS:0;e3d0adb87c01:40631 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/WALs/e3d0adb87c01,40631,1731831247805/e3d0adb87c01%2C40631%2C1731831247805.1731831251209, exclude list is [], retry=0 2024-11-17T08:14:11,227 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40497,DS-7eaf6057-7747-4a5f-8ff7-5300ff40dabc,DISK] 2024-11-17T08:14:11,230 INFO [RS:0;e3d0adb87c01:40631 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/WALs/e3d0adb87c01,40631,1731831247805/e3d0adb87c01%2C40631%2C1731831247805.1731831251209 2024-11-17T08:14:11,230 DEBUG [RS:0;e3d0adb87c01:40631 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42919:42919)] 2024-11-17T08:14:11,320 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-17T08:14:11,321 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85 2024-11-17T08:14:11,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741833_1009 (size=32) 2024-11-17T08:14:11,735 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T08:14:11,738 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-17T08:14:11,740 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-17T08:14:11,741 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:14:11,741 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T08:14:11,742 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-17T08:14:11,744 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-17T08:14:11,744 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:14:11,745 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T08:14:11,746 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-17T08:14:11,748 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-17T08:14:11,748 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:14:11,749 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T08:14:11,750 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-17T08:14:11,752 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-17T08:14:11,752 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:14:11,753 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T08:14:11,754 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-17T08:14:11,755 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/hbase/meta/1588230740 2024-11-17T08:14:11,756 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/hbase/meta/1588230740 2024-11-17T08:14:11,759 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-17T08:14:11,759 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-17T08:14:11,760 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T08:14:11,763 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-17T08:14:11,767 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T08:14:11,768 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72625365, jitterRate=0.08220227062702179}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T08:14:11,771 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731831251735Initializing all the Stores at 1731831251737 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731831251737Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731831251738 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731831251738Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731831251738Cleaning up temporary data from old regions at 1731831251759 (+21 ms)Region opened successfully at 1731831251771 (+12 ms) 2024-11-17T08:14:11,771 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-17T08:14:11,771 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-17T08:14:11,771 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-17T08:14:11,771 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-17T08:14:11,772 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-17T08:14:11,773 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-17T08:14:11,773 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731831251771Disabling compacts and flushes for region at 1731831251771Disabling writes for close at 1731831251772 (+1 ms)Writing region close event to WAL at 1731831251773 (+1 ms)Closed at 1731831251773 2024-11-17T08:14:11,776 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T08:14:11,776 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-17T08:14:11,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-17T08:14:11,790 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T08:14:11,793 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-17T08:14:11,952 DEBUG [e3d0adb87c01:45089 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-17T08:14:11,964 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:11,970 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e3d0adb87c01,40631,1731831247805, state=OPENING 2024-11-17T08:14:12,011 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-17T08:14:12,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45089-0x10147dba8ff0000, quorum=127.0.0.1:63364, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T08:14:12,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40631-0x10147dba8ff0001, quorum=127.0.0.1:63364, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T08:14:12,085 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T08:14:12,085 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T08:14:12,087 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T08:14:12,090 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e3d0adb87c01,40631,1731831247805}] 2024-11-17T08:14:12,267 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-17T08:14:12,270 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45171, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-17T08:14:12,280 INFO [RS_OPEN_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-17T08:14:12,280 INFO [RS_OPEN_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T08:14:12,281 INFO [RS_OPEN_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-17T08:14:12,283 INFO [RS_OPEN_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e3d0adb87c01%2C40631%2C1731831247805.meta, suffix=.meta, logDir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/WALs/e3d0adb87c01,40631,1731831247805, archiveDir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/oldWALs, maxLogs=32 2024-11-17T08:14:12,298 DEBUG [RS_OPEN_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/WALs/e3d0adb87c01,40631,1731831247805/e3d0adb87c01%2C40631%2C1731831247805.meta.1731831252285.meta, exclude list is [], retry=0 2024-11-17T08:14:12,301 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40497,DS-7eaf6057-7747-4a5f-8ff7-5300ff40dabc,DISK] 2024-11-17T08:14:12,304 INFO [RS_OPEN_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/WALs/e3d0adb87c01,40631,1731831247805/e3d0adb87c01%2C40631%2C1731831247805.meta.1731831252285.meta 2024-11-17T08:14:12,305 DEBUG [RS_OPEN_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42919:42919)] 2024-11-17T08:14:12,305 DEBUG [RS_OPEN_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-17T08:14:12,307 DEBUG [RS_OPEN_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-17T08:14:12,309 DEBUG [RS_OPEN_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-17T08:14:12,313 INFO [RS_OPEN_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-17T08:14:12,318 DEBUG [RS_OPEN_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-17T08:14:12,319 DEBUG [RS_OPEN_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T08:14:12,319 DEBUG [RS_OPEN_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-17T08:14:12,319 DEBUG [RS_OPEN_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-17T08:14:12,323 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-17T08:14:12,324 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-17T08:14:12,324 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:14:12,325 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T08:14:12,325 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-17T08:14:12,327 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-17T08:14:12,327 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:14:12,327 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T08:14:12,328 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-17T08:14:12,329 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-17T08:14:12,329 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:14:12,330 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T08:14:12,330 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-17T08:14:12,332 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-17T08:14:12,332 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:14:12,333 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T08:14:12,333 DEBUG [RS_OPEN_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-17T08:14:12,334 DEBUG [RS_OPEN_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/hbase/meta/1588230740 2024-11-17T08:14:12,337 DEBUG [RS_OPEN_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/hbase/meta/1588230740 2024-11-17T08:14:12,340 DEBUG [RS_OPEN_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-17T08:14:12,340 DEBUG [RS_OPEN_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-17T08:14:12,341 DEBUG [RS_OPEN_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T08:14:12,344 DEBUG [RS_OPEN_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-17T08:14:12,346 INFO [RS_OPEN_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61043968, jitterRate=-0.09037399291992188}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T08:14:12,346 DEBUG [RS_OPEN_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-17T08:14:12,348 DEBUG [RS_OPEN_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731831252320Writing region info on filesystem at 1731831252320Initializing all the Stores at 1731831252322 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731831252322Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731831252323 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731831252323Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731831252323Cleaning up temporary data from old regions at 1731831252340 (+17 ms)Running coprocessor post-open hooks at 1731831252346 (+6 ms)Region opened successfully at 1731831252348 (+2 ms) 2024-11-17T08:14:12,356 INFO [RS_OPEN_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731831252261 2024-11-17T08:14:12,366 DEBUG [RS_OPEN_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-17T08:14:12,367 INFO [RS_OPEN_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-17T08:14:12,368 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:12,370 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e3d0adb87c01,40631,1731831247805, state=OPEN 2024-11-17T08:14:12,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40631-0x10147dba8ff0001, quorum=127.0.0.1:63364, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T08:14:12,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45089-0x10147dba8ff0000, quorum=127.0.0.1:63364, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T08:14:12,472 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T08:14:12,472 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T08:14:12,473 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:12,482 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-17T08:14:12,482 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e3d0adb87c01,40631,1731831247805 in 384 msec 2024-11-17T08:14:12,488 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-17T08:14:12,489 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 702 msec 2024-11-17T08:14:12,490 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T08:14:12,490 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-17T08:14:12,509 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:14:12,510 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:14:12,527 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:14:12,529 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35615, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:14:12,548 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.7340 sec 2024-11-17T08:14:12,548 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731831252548, completionTime=-1 2024-11-17T08:14:12,550 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-17T08:14:12,551 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-17T08:14:12,574 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-17T08:14:12,574 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731831312574 2024-11-17T08:14:12,574 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731831372574 2024-11-17T08:14:12,575 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 24 msec 2024-11-17T08:14:12,577 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e3d0adb87c01,45089,1731831246985-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T08:14:12,577 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e3d0adb87c01,45089,1731831246985-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T08:14:12,577 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e3d0adb87c01,45089,1731831246985-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T08:14:12,578 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-e3d0adb87c01:45089, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T08:14:12,578 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-17T08:14:12,579 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-17T08:14:12,585 DEBUG [master/e3d0adb87c01:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-17T08:14:12,622 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.654sec 2024-11-17T08:14:12,624 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-17T08:14:12,626 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-17T08:14:12,627 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-17T08:14:12,627 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-17T08:14:12,627 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-17T08:14:12,628 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e3d0adb87c01,45089,1731831246985-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T08:14:12,629 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e3d0adb87c01,45089,1731831246985-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-17T08:14:12,637 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-17T08:14:12,638 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-17T08:14:12,638 INFO [master/e3d0adb87c01:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e3d0adb87c01,45089,1731831246985-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T08:14:12,714 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:12,715 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-17T08:14:12,715 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-17T08:14:12,718 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:14:12,720 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:14:12,728 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:14:12,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:14:12,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:14:12,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33bb444f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:12,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:14:12,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:14:12,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:14:12,737 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50076, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:14:12,739 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:12,740 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:14:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:14:12,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:14:12,749 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:14:12,766 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=e3d0adb87c01,45089,1731831246985 2024-11-17T08:14:12,777 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=230, ProcessCount=11, AvailableMemoryMB=2953 2024-11-17T08:14:12,784 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T08:14:12,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is e3d0adb87c01,45089,1731831246985 2024-11-17T08:14:12,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@404c298d 2024-11-17T08:14:12,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T08:14:12,797 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50084, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T08:14:12,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T08:14:12,803 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T08:14:12,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T08:14:12,814 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T08:14:12,816 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-17T08:14:12,817 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:14:12,819 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T08:14:12,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T08:14:12,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741835_1011 (size=1061) 2024-11-17T08:14:12,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T08:14:13,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T08:14:13,236 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85 2024-11-17T08:14:13,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741836_1012 (size=53) 2024-11-17T08:14:13,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T08:14:13,650 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T08:14:13,650 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 988129a9cf65d53c5913e9c54b149fd7, disabling compactions & flushes 2024-11-17T08:14:13,650 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:13,650 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:13,650 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. after waiting 0 ms 2024-11-17T08:14:13,650 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:13,650 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:13,651 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 988129a9cf65d53c5913e9c54b149fd7: Waiting for close lock at 1731831253650Disabling compacts and flushes for region at 1731831253650Disabling writes for close at 1731831253650Writing region close event to WAL at 1731831253650Closed at 1731831253650 2024-11-17T08:14:13,653 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T08:14:13,659 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731831253653"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731831253653"}]},"ts":"1731831253653"} 2024-11-17T08:14:13,670 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T08:14:13,673 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T08:14:13,676 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731831253673"}]},"ts":"1731831253673"} 2024-11-17T08:14:13,681 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T08:14:13,683 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=988129a9cf65d53c5913e9c54b149fd7, ASSIGN}] 2024-11-17T08:14:13,686 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=988129a9cf65d53c5913e9c54b149fd7, ASSIGN 2024-11-17T08:14:13,689 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=988129a9cf65d53c5913e9c54b149fd7, ASSIGN; state=OFFLINE, location=e3d0adb87c01,40631,1731831247805; forceNewPlan=false, retain=false 2024-11-17T08:14:13,841 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=988129a9cf65d53c5913e9c54b149fd7, regionState=OPENING, regionLocation=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:13,847 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=988129a9cf65d53c5913e9c54b149fd7, ASSIGN because future has completed 2024-11-17T08:14:13,848 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805}] 2024-11-17T08:14:13,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T08:14:14,011 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:14,011 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} 2024-11-17T08:14:14,012 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:14,012 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T08:14:14,012 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:14,012 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:14,015 INFO [StoreOpener-988129a9cf65d53c5913e9c54b149fd7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:14,020 INFO [StoreOpener-988129a9cf65d53c5913e9c54b149fd7-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T08:14:14,020 INFO [StoreOpener-988129a9cf65d53c5913e9c54b149fd7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 988129a9cf65d53c5913e9c54b149fd7 columnFamilyName A 2024-11-17T08:14:14,020 DEBUG [StoreOpener-988129a9cf65d53c5913e9c54b149fd7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:14:14,022 INFO [StoreOpener-988129a9cf65d53c5913e9c54b149fd7-1 {}] regionserver.HStore(327): Store=988129a9cf65d53c5913e9c54b149fd7/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T08:14:14,022 INFO [StoreOpener-988129a9cf65d53c5913e9c54b149fd7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:14,024 INFO [StoreOpener-988129a9cf65d53c5913e9c54b149fd7-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T08:14:14,024 INFO [StoreOpener-988129a9cf65d53c5913e9c54b149fd7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 988129a9cf65d53c5913e9c54b149fd7 columnFamilyName B 2024-11-17T08:14:14,025 DEBUG [StoreOpener-988129a9cf65d53c5913e9c54b149fd7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:14:14,025 INFO [StoreOpener-988129a9cf65d53c5913e9c54b149fd7-1 {}] regionserver.HStore(327): Store=988129a9cf65d53c5913e9c54b149fd7/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T08:14:14,026 INFO [StoreOpener-988129a9cf65d53c5913e9c54b149fd7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:14,028 INFO [StoreOpener-988129a9cf65d53c5913e9c54b149fd7-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T08:14:14,028 INFO [StoreOpener-988129a9cf65d53c5913e9c54b149fd7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 988129a9cf65d53c5913e9c54b149fd7 columnFamilyName C 2024-11-17T08:14:14,028 DEBUG [StoreOpener-988129a9cf65d53c5913e9c54b149fd7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:14:14,029 INFO [StoreOpener-988129a9cf65d53c5913e9c54b149fd7-1 {}] regionserver.HStore(327): Store=988129a9cf65d53c5913e9c54b149fd7/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T08:14:14,030 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:14,030 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:14,032 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:14,033 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:14,035 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:14,035 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:14,036 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T08:14:14,038 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:14,041 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T08:14:14,042 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 988129a9cf65d53c5913e9c54b149fd7; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61461010, jitterRate=-0.08415958285331726}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T08:14:14,042 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:14,043 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 988129a9cf65d53c5913e9c54b149fd7: Running coprocessor pre-open hook at 1731831254013Writing region info on filesystem at 1731831254013Initializing all the Stores at 1731831254015 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731831254015Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731831254015Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731831254015Cleaning up temporary data from old regions at 1731831254035 (+20 ms)Running coprocessor post-open hooks at 1731831254042 (+7 ms)Region opened successfully at 1731831254043 (+1 ms) 2024-11-17T08:14:14,046 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., pid=6, masterSystemTime=1731831254003 2024-11-17T08:14:14,050 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:14,050 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:14,051 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=988129a9cf65d53c5913e9c54b149fd7, regionState=OPEN, openSeqNum=2, regionLocation=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:14,055 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 because future has completed 2024-11-17T08:14:14,060 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-17T08:14:14,060 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 in 208 msec 2024-11-17T08:14:14,064 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-17T08:14:14,064 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=988129a9cf65d53c5913e9c54b149fd7, ASSIGN in 377 msec 2024-11-17T08:14:14,065 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T08:14:14,066 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731831254065"}]},"ts":"1731831254065"} 2024-11-17T08:14:14,069 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T08:14:14,070 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T08:14:14,073 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2640 sec 2024-11-17T08:14:14,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T08:14:14,976 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:14:14,985 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:14,985 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:14:14,986 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:14:14,987 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:14:14,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:14:14,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:14:14,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60daaef1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:14,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:14:14,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:14:14,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:14:14,990 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:14:14,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bebe1d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:14,993 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:14:14,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:14:14,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:14:14,996 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40692, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:14:14,998 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T08:14:15,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is e3d0adb87c01,45089,1731831246985 2024-11-17T08:14:15,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3a1cfc3c 2024-11-17T08:14:15,002 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T08:14:15,005 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50112, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T08:14:15,009 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:15,009 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:14:15,009 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:14:15,011 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:14:15,011 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:14:15,011 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:14:15,012 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@703ed473, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:15,012 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:14:15,012 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:14:15,012 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:14:15,014 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:14:15,015 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3937ace3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:15,016 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:15,016 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:14:15,016 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:14:15,017 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:14:15,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:14:15,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:14:15,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ffd0298, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:15,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:14:15,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:14:15,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:14:15,020 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50154, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:14:15,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f55d533, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:15,023 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:15,023 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:14:15,023 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:14:15,024 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:14:15,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:14:15,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:14:15,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d1d7645, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:15,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:14:15,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:14:15,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:14:15,029 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50172, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:14:15,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10bc6ecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:15,031 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:15,031 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:14:15,031 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:14:15,033 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:14:15,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:14:15,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:14:15,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e153ddc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:15,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:14:15,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:14:15,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:14:15,036 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:14:15,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35159008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:15,037 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:15,038 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:14:15,038 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:14:15,039 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:14:15,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:14:15,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:14:15,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52d03cf6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:15,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:14:15,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:14:15,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:14:15,041 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:14:15,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47cbee8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:15,043 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:15,043 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:14:15,044 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:14:15,045 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:14:15,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:14:15,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:14:15,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d6e4493, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:15,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:14:15,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:14:15,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:14:15,047 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50220, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:14:15,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1455c669, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:15,049 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:15,049 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:14:15,049 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:14:15,050 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:14:15,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:14:15,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:14:15,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75b90827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:15,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:14:15,052 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:14:15,052 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:14:15,053 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50232, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:14:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fc9366a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:15,055 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:15,055 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:14:15,055 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:14:15,056 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:14:15,056 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:14:15,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:14:15,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f11d0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:15,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:14:15,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:14:15,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:14:15,058 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50242, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:14:15,059 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b92dc16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:15,060 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:15,061 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:14:15,061 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:14:15,062 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:14:15,062 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:14:15,062 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:14:15,063 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c83f8e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:15,063 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:14:15,063 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:14:15,063 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:14:15,064 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50262, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:14:15,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c944594, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:15,069 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:14:15,069 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:14:15,069 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:14:15,069 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:14:15,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:14:15,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:14:15,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:14:15,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:14:15,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:14:15,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:14:15,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:14:15,073 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:14:15,073 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:14:15,074 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:14:15,074 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:14:15,075 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:14:15,076 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40714, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:14:15,076 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:14:15,078 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:14:15,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:14:15,080 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40724, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:14:15,079 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:14:15,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:14:15,082 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40742, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:14:15,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2] 2024-11-17T08:14:15,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2] 2024-11-17T08:14:15,085 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:14:15,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:14:15,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2] 2024-11-17T08:14:15,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:14:15,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2] 2024-11-17T08:14:15,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2] 2024-11-17T08:14:15,093 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:14:15,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2] 2024-11-17T08:14:15,095 DEBUG [Thread-151 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:14:15,095 DEBUG [Thread-151 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:14:15,099 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:14:15,100 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:14:15,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2] 2024-11-17T08:14:15,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2] 2024-11-17T08:14:15,108 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:14:15,109 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:14:15,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:14:15,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:14:15,115 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:14:15,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2] 2024-11-17T08:14:15,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-17T08:14:15,122 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:14:15,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T08:14:15,125 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:14:15,126 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:14:15,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:15,172 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 988129a9cf65d53c5913e9c54b149fd7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T08:14:15,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=A 2024-11-17T08:14:15,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:15,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=B 2024-11-17T08:14:15,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:15,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=C 2024-11-17T08:14:15,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:15,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T08:14:15,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T08:14:15,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:15,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:15,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:15,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:15,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:15,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:15,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/443720ad1e2f4e79a63de18962dbb31a is 50, key is test_row_0/A:col10/1731831255117/Put/seqid=0 2024-11-17T08:14:15,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741837_1013 (size=16681) 2024-11-17T08:14:15,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:15,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731831255340, max timestamp=1731831255375Num uniques 859; 2024-11-17T08:14:15,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731831255340, max timestamp=1731831255375Num uniques 859; 2024-11-17T08:14:15,411 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T08:14:15,411 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T08:14:15,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731831255340, max timestamp=1731831255379Num uniques 859; 2024-11-17T08:14:15,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T08:14:15,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831315378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:15,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T08:14:15,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:15,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:15,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:15,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T08:14:15,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:15,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:15,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:15,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:15,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:15,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:15,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:15,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831315503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:15,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:15,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:15,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831315517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:15,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:15,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:15,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:15,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:15,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:15,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:15,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831315524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:15,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:15,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:15,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:15,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:15,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831315528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:15,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:15,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:15,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:15,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:15,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831315564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:15,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:15,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:15,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831315635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:15,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:15,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:15,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:15,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:15,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831315635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:15,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T08:14:15,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:15,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:15,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:15,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:15,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:15,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:15,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:15,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831315645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:15,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:15,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:15,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:15,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:15,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831315646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:15,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:15,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:15,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:15,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T08:14:15,772 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/443720ad1e2f4e79a63de18962dbb31a 2024-11-17T08:14:15,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:15,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831315774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:15,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:15,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:15,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:15,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T08:14:15,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:15,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:15,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:15,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:15,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:15,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:15,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:15,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831315844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:15,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:15,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:15,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:15,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:15,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831315845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:15,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:15,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:15,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:15,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:15,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831315864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:15,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:15,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:15,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:15,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:15,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831315864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:15,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:15,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:15,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:15,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/dfaa20c2bce24f978d2763b23894fa7e is 50, key is test_row_0/B:col10/1731831255117/Put/seqid=0 2024-11-17T08:14:15,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741838_1014 (size=12001) 2024-11-17T08:14:15,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/dfaa20c2bce24f978d2763b23894fa7e 2024-11-17T08:14:15,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/62fbc9b549574e5db518e89c9c6b80dc is 50, key is test_row_0/C:col10/1731831255117/Put/seqid=0 2024-11-17T08:14:15,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T08:14:15,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:15,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:15,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:15,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:15,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:15,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:15,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741839_1015 (size=12001) 2024-11-17T08:14:15,976 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/62fbc9b549574e5db518e89c9c6b80dc 2024-11-17T08:14:16,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/443720ad1e2f4e79a63de18962dbb31a as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/443720ad1e2f4e79a63de18962dbb31a 2024-11-17T08:14:16,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/443720ad1e2f4e79a63de18962dbb31a, entries=250, sequenceid=12, filesize=16.3 K 2024-11-17T08:14:16,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/dfaa20c2bce24f978d2763b23894fa7e as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/dfaa20c2bce24f978d2763b23894fa7e 2024-11-17T08:14:16,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/dfaa20c2bce24f978d2763b23894fa7e, entries=150, sequenceid=12, filesize=11.7 K 2024-11-17T08:14:16,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/62fbc9b549574e5db518e89c9c6b80dc as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/62fbc9b549574e5db518e89c9c6b80dc 2024-11-17T08:14:16,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/62fbc9b549574e5db518e89c9c6b80dc, entries=150, sequenceid=12, filesize=11.7 K 2024-11-17T08:14:16,087 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 988129a9cf65d53c5913e9c54b149fd7 in 915ms, sequenceid=12, compaction requested=false 2024-11-17T08:14:16,089 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-17T08:14:16,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:16,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T08:14:16,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:16,121 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 988129a9cf65d53c5913e9c54b149fd7 3/3 column families, dataSize=134.18 KB heapSize=385.45 KB 2024-11-17T08:14:16,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:16,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:16,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=A 2024-11-17T08:14:16,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T08:14:16,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=B 2024-11-17T08:14:16,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T08:14:16,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=C 2024-11-17T08:14:16,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T08:14:16,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/36655c9ea9fb457dbf6eabfbf1188845 is 50, key is test_row_0/A:col10/1731831255518/Put/seqid=0 2024-11-17T08:14:16,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741840_1016 (size=14341) 2024-11-17T08:14:16,204 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/36655c9ea9fb457dbf6eabfbf1188845 2024-11-17T08:14:16,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:16,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831316192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:16,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:16,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:16,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:16,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/814237c689d249bfb3a8656fe8e035dc is 50, key is test_row_0/B:col10/1731831255518/Put/seqid=0 2024-11-17T08:14:16,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741841_1017 (size=12001) 2024-11-17T08:14:16,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:16,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831316228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:16,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:16,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:16,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:16,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T08:14:16,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:16,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831316233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:16,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:16,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831316237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:16,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:16,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831316241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:16,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:16,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:16,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:16,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:16,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:16,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:16,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:16,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:16,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:16,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:16,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831316345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:16,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:16,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:16,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:16,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:16,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831316375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:16,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:16,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:16,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:16,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:16,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831316385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:16,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:16,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831316386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:16,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:16,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:16,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:16,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:16,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831316387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:16,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:16,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:16,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:16,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:16,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:16,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:16,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:16,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831316564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:16,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:16,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:16,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:16,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:16,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831316604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:16,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:16,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:16,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:16,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:16,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831316615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:16,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:16,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831316617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:16,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:16,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831316617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:16,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:16,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:16,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:16,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:16,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:16,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:16,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:16,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:16,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:16,663 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/814237c689d249bfb3a8656fe8e035dc 2024-11-17T08:14:16,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/28bdbd75f4d744119b8531e7cee65de4 is 50, key is test_row_0/C:col10/1731831255518/Put/seqid=0 2024-11-17T08:14:16,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741842_1018 (size=12001) 2024-11-17T08:14:16,712 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/28bdbd75f4d744119b8531e7cee65de4 2024-11-17T08:14:16,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/36655c9ea9fb457dbf6eabfbf1188845 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/36655c9ea9fb457dbf6eabfbf1188845 2024-11-17T08:14:16,750 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/36655c9ea9fb457dbf6eabfbf1188845, entries=200, sequenceid=50, filesize=14.0 K 2024-11-17T08:14:16,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/814237c689d249bfb3a8656fe8e035dc as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/814237c689d249bfb3a8656fe8e035dc 2024-11-17T08:14:16,780 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/814237c689d249bfb3a8656fe8e035dc, entries=150, sequenceid=50, filesize=11.7 K 2024-11-17T08:14:16,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/28bdbd75f4d744119b8531e7cee65de4 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/28bdbd75f4d744119b8531e7cee65de4 2024-11-17T08:14:16,809 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/28bdbd75f4d744119b8531e7cee65de4, entries=150, sequenceid=50, filesize=11.7 K 2024-11-17T08:14:16,813 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=46.96 KB/48090 for 988129a9cf65d53c5913e9c54b149fd7 in 691ms, sequenceid=50, compaction requested=false 2024-11-17T08:14:16,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:16,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:16,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-17T08:14:16,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-17T08:14:16,830 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-17T08:14:16,831 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6990 sec 2024-11-17T08:14:16,841 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.7220 sec 2024-11-17T08:14:16,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:16,891 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 988129a9cf65d53c5913e9c54b149fd7 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-17T08:14:16,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=A 2024-11-17T08:14:16,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:16,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=B 2024-11-17T08:14:16,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:16,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=C 2024-11-17T08:14:16,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:16,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/5da72974eb5b4f81ba9784a9733993a8 is 50, key is test_row_0/A:col10/1731831256227/Put/seqid=0 2024-11-17T08:14:16,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741843_1019 (size=11997) 2024-11-17T08:14:16,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/5da72974eb5b4f81ba9784a9733993a8 2024-11-17T08:14:16,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/c8dad1a3c6d9455986a88d7eabf87b73 is 50, key is test_row_0/B:col10/1731831256227/Put/seqid=0 2024-11-17T08:14:16,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741844_1020 (size=9657) 2024-11-17T08:14:16,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/c8dad1a3c6d9455986a88d7eabf87b73 2024-11-17T08:14:17,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731831256964, max timestamp=1731831256989Num uniques 859; 2024-11-17T08:14:17,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731831256964, max timestamp=1731831256993Num uniques 859; 2024-11-17T08:14:17,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T08:14:17,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T08:14:17,011 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-17T08:14:17,012 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731831256964, max timestamp=1731831256989Num uniques 859; 2024-11-17T08:14:17,012 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T08:14:17,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/946a3fcc9ad24a6c9b13d04e76b850ce is 50, key is test_row_0/C:col10/1731831256227/Put/seqid=0 2024-11-17T08:14:17,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741845_1021 (size=9657) 2024-11-17T08:14:17,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/946a3fcc9ad24a6c9b13d04e76b850ce 2024-11-17T08:14:17,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/5da72974eb5b4f81ba9784a9733993a8 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/5da72974eb5b4f81ba9784a9733993a8 2024-11-17T08:14:17,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:17,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831317049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:17,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:17,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:17,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831317054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:17,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:17,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:17,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831317057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:17,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:17,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:17,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831317065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:17,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:17,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831317067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:17,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:17,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/5da72974eb5b4f81ba9784a9733993a8, entries=150, sequenceid=61, filesize=11.7 K 2024-11-17T08:14:17,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:17,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/c8dad1a3c6d9455986a88d7eabf87b73 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/c8dad1a3c6d9455986a88d7eabf87b73 2024-11-17T08:14:17,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/c8dad1a3c6d9455986a88d7eabf87b73, entries=100, sequenceid=61, filesize=9.4 K 2024-11-17T08:14:17,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/946a3fcc9ad24a6c9b13d04e76b850ce as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/946a3fcc9ad24a6c9b13d04e76b850ce 2024-11-17T08:14:17,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/946a3fcc9ad24a6c9b13d04e76b850ce, entries=100, sequenceid=61, filesize=9.4 K 2024-11-17T08:14:17,114 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 988129a9cf65d53c5913e9c54b149fd7 in 222ms, sequenceid=61, compaction requested=true 2024-11-17T08:14:17,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:17,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 988129a9cf65d53c5913e9c54b149fd7:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:14:17,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:17,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 988129a9cf65d53c5913e9c54b149fd7:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:14:17,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:17,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 988129a9cf65d53c5913e9c54b149fd7:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:14:17,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:14:17,125 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:14:17,127 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:14:17,131 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:14:17,132 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): 988129a9cf65d53c5913e9c54b149fd7/B is initiating minor compaction (all files) 2024-11-17T08:14:17,133 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 988129a9cf65d53c5913e9c54b149fd7/B in TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:17,133 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/dfaa20c2bce24f978d2763b23894fa7e, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/814237c689d249bfb3a8656fe8e035dc, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/c8dad1a3c6d9455986a88d7eabf87b73] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp, totalSize=32.9 K 2024-11-17T08:14:17,135 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43019 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:14:17,135 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): 988129a9cf65d53c5913e9c54b149fd7/A is initiating minor compaction (all files) 2024-11-17T08:14:17,135 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 988129a9cf65d53c5913e9c54b149fd7/A in TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:17,136 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/443720ad1e2f4e79a63de18962dbb31a, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/36655c9ea9fb457dbf6eabfbf1188845, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/5da72974eb5b4f81ba9784a9733993a8] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp, totalSize=42.0 K 2024-11-17T08:14:17,137 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting dfaa20c2bce24f978d2763b23894fa7e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731831255117 2024-11-17T08:14:17,137 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 443720ad1e2f4e79a63de18962dbb31a, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731831255117 2024-11-17T08:14:17,138 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36655c9ea9fb457dbf6eabfbf1188845, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731831255468 2024-11-17T08:14:17,138 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 814237c689d249bfb3a8656fe8e035dc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731831255468 2024-11-17T08:14:17,139 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5da72974eb5b4f81ba9784a9733993a8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731831256227 2024-11-17T08:14:17,139 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting c8dad1a3c6d9455986a88d7eabf87b73, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731831256227 2024-11-17T08:14:17,177 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 988129a9cf65d53c5913e9c54b149fd7#A#compaction#9 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:14:17,178 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/398cdee7f53e424b97c01b7dfdd9021d is 50, key is test_row_0/A:col10/1731831256227/Put/seqid=0 2024-11-17T08:14:17,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:17,181 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 988129a9cf65d53c5913e9c54b149fd7#B#compaction#10 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:14:17,183 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/cd74d8753c674dc1aac54f3d44c0ab40 is 50, key is test_row_0/B:col10/1731831256227/Put/seqid=0 2024-11-17T08:14:17,184 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731831257050, max timestamp=1731831257174Num uniques 1018; 2024-11-17T08:14:17,184 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731831257050, max timestamp=1731831257174Num uniques 1018; 2024-11-17T08:14:17,184 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-17T08:14:17,184 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-17T08:14:17,185 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 988129a9cf65d53c5913e9c54b149fd7 3/3 column families, dataSize=71.56 KB heapSize=198.72 KB 2024-11-17T08:14:17,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=A 2024-11-17T08:14:17,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T08:14:17,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=B 2024-11-17T08:14:17,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:14:17,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=C 2024-11-17T08:14:17,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:14:17,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741846_1022 (size=12104) 2024-11-17T08:14:17,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/0f18a73db9b54237ad0c17b8d64ea7c6 is 50, key is test_row_0/A:col10/1731831257184/Put/seqid=0 2024-11-17T08:14:17,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741847_1023 (size=12104) 2024-11-17T08:14:17,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741848_1024 (size=19021) 2024-11-17T08:14:17,238 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/0f18a73db9b54237ad0c17b8d64ea7c6 2024-11-17T08:14:17,252 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/cd74d8753c674dc1aac54f3d44c0ab40 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/cd74d8753c674dc1aac54f3d44c0ab40 2024-11-17T08:14:17,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/495a8c51764b4566a9c19a85a8c2ed51 is 50, key is test_row_0/B:col10/1731831257184/Put/seqid=0 2024-11-17T08:14:17,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T08:14:17,284 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:14:17,285 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 988129a9cf65d53c5913e9c54b149fd7/B of 988129a9cf65d53c5913e9c54b149fd7 into cd74d8753c674dc1aac54f3d44c0ab40(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:14:17,285 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:17,286 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:14:17,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-17T08:14:17,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T08:14:17,289 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., storeName=988129a9cf65d53c5913e9c54b149fd7/B, priority=13, startTime=1731831257123; duration=0sec 2024-11-17T08:14:17,289 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:14:17,290 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:14:17,290 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 988129a9cf65d53c5913e9c54b149fd7:B 2024-11-17T08:14:17,290 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:14:17,291 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:14:17,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:14:17,293 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:14:17,293 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): 988129a9cf65d53c5913e9c54b149fd7/C is initiating minor compaction (all files) 2024-11-17T08:14:17,293 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 988129a9cf65d53c5913e9c54b149fd7/C in TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:17,294 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/62fbc9b549574e5db518e89c9c6b80dc, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/28bdbd75f4d744119b8531e7cee65de4, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/946a3fcc9ad24a6c9b13d04e76b850ce] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp, totalSize=32.9 K 2024-11-17T08:14:17,295 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 62fbc9b549574e5db518e89c9c6b80dc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731831255117 2024-11-17T08:14:17,296 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 28bdbd75f4d744119b8531e7cee65de4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731831255468 2024-11-17T08:14:17,297 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 946a3fcc9ad24a6c9b13d04e76b850ce, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731831256227 2024-11-17T08:14:17,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:17,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741849_1025 (size=12001) 2024-11-17T08:14:17,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831317280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:17,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:17,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:17,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831317281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:17,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:17,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831317283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:17,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:17,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:17,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:17,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831317288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:17,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:17,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:17,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831317300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:17,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:17,329 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 988129a9cf65d53c5913e9c54b149fd7#C#compaction#13 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:14:17,330 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/a49cb8bbdd9e4e0b8b29fc9e92722869 is 50, key is test_row_0/C:col10/1731831256227/Put/seqid=0 2024-11-17T08:14:17,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741850_1026 (size=12104) 2024-11-17T08:14:17,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T08:14:17,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:17,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831317404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:17,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:17,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:17,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831317414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:17,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:17,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:17,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831317415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:17,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:17,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:17,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831317415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:17,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:17,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831317415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:17,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:17,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:17,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-17T08:14:17,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:17,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:17,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:17,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:17,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:17,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:17,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-17T08:14:17,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:17,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:17,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:17,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:17,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:17,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:17,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T08:14:17,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:17,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831317614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:17,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:17,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:17,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831317625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:17,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:17,631 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/398cdee7f53e424b97c01b7dfdd9021d as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/398cdee7f53e424b97c01b7dfdd9021d 2024-11-17T08:14:17,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:17,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831317636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:17,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:17,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:17,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831317637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:17,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:17,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:17,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831317637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:17,646 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 988129a9cf65d53c5913e9c54b149fd7/A of 988129a9cf65d53c5913e9c54b149fd7 into 398cdee7f53e424b97c01b7dfdd9021d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:14:17,647 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:17,647 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., storeName=988129a9cf65d53c5913e9c54b149fd7/A, priority=13, startTime=1731831257117; duration=0sec 2024-11-17T08:14:17,647 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:17,647 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 988129a9cf65d53c5913e9c54b149fd7:A 2024-11-17T08:14:17,704 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/495a8c51764b4566a9c19a85a8c2ed51 2024-11-17T08:14:17,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/e0a24c7a91fe43dd88155f4f8c00f247 is 50, key is test_row_0/C:col10/1731831257184/Put/seqid=0 2024-11-17T08:14:17,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-17T08:14:17,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:17,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:17,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:17,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:17,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:17,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:17,766 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/a49cb8bbdd9e4e0b8b29fc9e92722869 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/a49cb8bbdd9e4e0b8b29fc9e92722869 2024-11-17T08:14:17,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741851_1027 (size=12001) 2024-11-17T08:14:17,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/e0a24c7a91fe43dd88155f4f8c00f247 2024-11-17T08:14:17,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/0f18a73db9b54237ad0c17b8d64ea7c6 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/0f18a73db9b54237ad0c17b8d64ea7c6 2024-11-17T08:14:17,796 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/0f18a73db9b54237ad0c17b8d64ea7c6, entries=300, sequenceid=101, filesize=18.6 K 2024-11-17T08:14:17,798 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 988129a9cf65d53c5913e9c54b149fd7/C of 988129a9cf65d53c5913e9c54b149fd7 into a49cb8bbdd9e4e0b8b29fc9e92722869(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:14:17,799 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:17,799 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., storeName=988129a9cf65d53c5913e9c54b149fd7/C, priority=13, startTime=1731831257123; duration=0sec 2024-11-17T08:14:17,799 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:17,799 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 988129a9cf65d53c5913e9c54b149fd7:C 2024-11-17T08:14:17,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/495a8c51764b4566a9c19a85a8c2ed51 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/495a8c51764b4566a9c19a85a8c2ed51 2024-11-17T08:14:17,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/495a8c51764b4566a9c19a85a8c2ed51, entries=150, sequenceid=101, filesize=11.7 K 2024-11-17T08:14:17,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/e0a24c7a91fe43dd88155f4f8c00f247 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/e0a24c7a91fe43dd88155f4f8c00f247 2024-11-17T08:14:17,832 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T08:14:17,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/e0a24c7a91fe43dd88155f4f8c00f247, entries=150, sequenceid=101, filesize=11.7 K 2024-11-17T08:14:17,839 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~78.27 KB/80150, heapSize ~218.59 KB/223840, currentSize=100.63 KB/103050 for 988129a9cf65d53c5913e9c54b149fd7 in 655ms, sequenceid=101, compaction requested=false 2024-11-17T08:14:17,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:17,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T08:14:17,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-17T08:14:17,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:17,916 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 988129a9cf65d53c5913e9c54b149fd7 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-17T08:14:17,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=A 2024-11-17T08:14:17,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:17,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=B 2024-11-17T08:14:17,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:17,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=C 2024-11-17T08:14:17,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:17,925 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-17T08:14:17,925 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-17T08:14:17,929 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-17T08:14:17,929 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-17T08:14:17,929 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-17T08:14:17,929 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-17T08:14:17,931 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-17T08:14:17,931 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-17T08:14:17,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/74aad59bc1504c99b5fe215a41ad0246 is 50, key is test_row_0/A:col10/1731831257282/Put/seqid=0 2024-11-17T08:14:17,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:17,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:17,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741852_1028 (size=12001) 2024-11-17T08:14:17,969 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/74aad59bc1504c99b5fe215a41ad0246 2024-11-17T08:14:17,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:17,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831317963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:17,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:17,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:17,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831317967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:17,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:17,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:17,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831317967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:17,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:17,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:17,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831317972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:17,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:17,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831317982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:17,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:17,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:17,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:17,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/dfa2e3b99438442ab7e83478f39e50e1 is 50, key is test_row_0/B:col10/1731831257282/Put/seqid=0 2024-11-17T08:14:18,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741853_1029 (size=12001) 2024-11-17T08:14:18,020 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/dfa2e3b99438442ab7e83478f39e50e1 2024-11-17T08:14:18,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/eb1463a46c434432aa1c943de9e6d849 is 50, key is test_row_0/C:col10/1731831257282/Put/seqid=0 2024-11-17T08:14:18,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741854_1030 (size=12001) 2024-11-17T08:14:18,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:18,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:18,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831318096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:18,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831318096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:18,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:18,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:18,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:18,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:18,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831318096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:18,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:18,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831318096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:18,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:18,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:18,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:18,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:18,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:18,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:18,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:18,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831318096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:18,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:18,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:18,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:18,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:18,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:18,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:18,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831318303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:18,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:18,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:18,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:18,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:18,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831318304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:18,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:18,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:18,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:18,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831318304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:18,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:18,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:18,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:18,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:18,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831318304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:18,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:18,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:18,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:18,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:18,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831318313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:18,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:18,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:18,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:18,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T08:14:18,459 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/eb1463a46c434432aa1c943de9e6d849 2024-11-17T08:14:18,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/74aad59bc1504c99b5fe215a41ad0246 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/74aad59bc1504c99b5fe215a41ad0246 2024-11-17T08:14:18,484 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/74aad59bc1504c99b5fe215a41ad0246, entries=150, sequenceid=122, filesize=11.7 K 2024-11-17T08:14:18,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/dfa2e3b99438442ab7e83478f39e50e1 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/dfa2e3b99438442ab7e83478f39e50e1 2024-11-17T08:14:18,511 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/dfa2e3b99438442ab7e83478f39e50e1, entries=150, sequenceid=122, filesize=11.7 K 2024-11-17T08:14:18,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/eb1463a46c434432aa1c943de9e6d849 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/eb1463a46c434432aa1c943de9e6d849 2024-11-17T08:14:18,525 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/eb1463a46c434432aa1c943de9e6d849, entries=150, sequenceid=122, filesize=11.7 K 2024-11-17T08:14:18,527 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 988129a9cf65d53c5913e9c54b149fd7 in 611ms, sequenceid=122, compaction requested=true 2024-11-17T08:14:18,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:18,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:18,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-17T08:14:18,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-17T08:14:18,533 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-17T08:14:18,533 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2390 sec 2024-11-17T08:14:18,538 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.2480 sec 2024-11-17T08:14:18,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:18,616 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 988129a9cf65d53c5913e9c54b149fd7 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-17T08:14:18,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=A 2024-11-17T08:14:18,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:18,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=B 2024-11-17T08:14:18,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:18,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=C 2024-11-17T08:14:18,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:18,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/a4e397e272dc477ab3d60994c677c614 is 50, key is test_row_0/A:col10/1731831258614/Put/seqid=0 2024-11-17T08:14:18,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741855_1031 (size=14541) 2024-11-17T08:14:18,644 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/a4e397e272dc477ab3d60994c677c614 2024-11-17T08:14:18,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/703b6f009d6344c199a887c6c72cb683 is 50, key is test_row_0/B:col10/1731831258614/Put/seqid=0 2024-11-17T08:14:18,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741856_1032 (size=12151) 2024-11-17T08:14:18,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:18,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831318690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:18,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:18,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:18,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:18,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:18,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831318690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:18,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:18,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:18,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:18,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:18,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831318690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:18,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:18,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:18,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:18,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:18,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831318694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:18,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:18,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:18,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:18,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831318694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:18,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:18,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:18,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:18,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:18,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:18,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:18,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831318803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:18,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831318804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:18,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:18,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:18,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:18,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:18,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831318804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:18,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:18,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:18,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:18,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:18,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:18,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:18,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:18,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831318814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:18,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:18,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:18,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:18,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:18,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831318814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:18,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:18,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:18,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:19,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:19,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831319014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:19,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:19,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:19,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:19,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:19,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831319015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:19,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:19,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:19,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:19,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:19,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831319024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:19,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:19,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:19,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:19,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:19,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831319025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:19,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:19,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:19,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:19,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:19,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831319025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:19,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:19,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:19,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:19,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/703b6f009d6344c199a887c6c72cb683 2024-11-17T08:14:19,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/c08eafd1f1c4414fbcf21d1e2d731e14 is 50, key is test_row_0/C:col10/1731831258614/Put/seqid=0 2024-11-17T08:14:19,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741857_1033 (size=12151) 2024-11-17T08:14:19,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/c08eafd1f1c4414fbcf21d1e2d731e14 2024-11-17T08:14:19,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/a4e397e272dc477ab3d60994c677c614 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/a4e397e272dc477ab3d60994c677c614 2024-11-17T08:14:19,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/a4e397e272dc477ab3d60994c677c614, entries=200, sequenceid=137, filesize=14.2 K 2024-11-17T08:14:19,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/703b6f009d6344c199a887c6c72cb683 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/703b6f009d6344c199a887c6c72cb683 2024-11-17T08:14:19,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/703b6f009d6344c199a887c6c72cb683, entries=150, sequenceid=137, filesize=11.9 K 2024-11-17T08:14:19,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/c08eafd1f1c4414fbcf21d1e2d731e14 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/c08eafd1f1c4414fbcf21d1e2d731e14 2024-11-17T08:14:19,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/c08eafd1f1c4414fbcf21d1e2d731e14, entries=150, sequenceid=137, filesize=11.9 K 2024-11-17T08:14:19,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 988129a9cf65d53c5913e9c54b149fd7 in 566ms, sequenceid=137, compaction requested=true 2024-11-17T08:14:19,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:19,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 988129a9cf65d53c5913e9c54b149fd7:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:14:19,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:19,183 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T08:14:19,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 988129a9cf65d53c5913e9c54b149fd7:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:14:19,183 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T08:14:19,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:19,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 988129a9cf65d53c5913e9c54b149fd7:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:14:19,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:14:19,189 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T08:14:19,189 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57667 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T08:14:19,189 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): 988129a9cf65d53c5913e9c54b149fd7/B is initiating minor compaction (all files) 2024-11-17T08:14:19,189 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): 988129a9cf65d53c5913e9c54b149fd7/A is initiating minor compaction (all files) 2024-11-17T08:14:19,189 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 988129a9cf65d53c5913e9c54b149fd7/B in TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:19,189 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 988129a9cf65d53c5913e9c54b149fd7/A in TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:19,190 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/cd74d8753c674dc1aac54f3d44c0ab40, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/495a8c51764b4566a9c19a85a8c2ed51, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/dfa2e3b99438442ab7e83478f39e50e1, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/703b6f009d6344c199a887c6c72cb683] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp, totalSize=47.1 K 2024-11-17T08:14:19,190 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/398cdee7f53e424b97c01b7dfdd9021d, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/0f18a73db9b54237ad0c17b8d64ea7c6, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/74aad59bc1504c99b5fe215a41ad0246, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/a4e397e272dc477ab3d60994c677c614] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp, totalSize=56.3 K 2024-11-17T08:14:19,192 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 398cdee7f53e424b97c01b7dfdd9021d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731831256109 2024-11-17T08:14:19,192 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting cd74d8753c674dc1aac54f3d44c0ab40, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731831256109 2024-11-17T08:14:19,193 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f18a73db9b54237ad0c17b8d64ea7c6, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731831257050 2024-11-17T08:14:19,193 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 495a8c51764b4566a9c19a85a8c2ed51, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731831257061 2024-11-17T08:14:19,194 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74aad59bc1504c99b5fe215a41ad0246, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731831257279 2024-11-17T08:14:19,194 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting dfa2e3b99438442ab7e83478f39e50e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731831257279 2024-11-17T08:14:19,194 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4e397e272dc477ab3d60994c677c614, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731831257962 2024-11-17T08:14:19,195 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 703b6f009d6344c199a887c6c72cb683, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731831257962 2024-11-17T08:14:19,224 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 988129a9cf65d53c5913e9c54b149fd7#A#compaction#21 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:14:19,225 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/975ccd3742eb429ca765486bfc976e87 is 50, key is test_row_0/A:col10/1731831258614/Put/seqid=0 2024-11-17T08:14:19,227 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 988129a9cf65d53c5913e9c54b149fd7#B#compaction#22 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:14:19,229 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/04955baeea824b67b1a78d23d398f3e1 is 50, key is test_row_0/B:col10/1731831258614/Put/seqid=0 2024-11-17T08:14:19,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741859_1035 (size=12391) 2024-11-17T08:14:19,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741858_1034 (size=12391) 2024-11-17T08:14:19,298 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/975ccd3742eb429ca765486bfc976e87 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/975ccd3742eb429ca765486bfc976e87 2024-11-17T08:14:19,313 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 988129a9cf65d53c5913e9c54b149fd7/A of 988129a9cf65d53c5913e9c54b149fd7 into 975ccd3742eb429ca765486bfc976e87(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:14:19,313 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:19,313 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., storeName=988129a9cf65d53c5913e9c54b149fd7/A, priority=12, startTime=1731831259183; duration=0sec 2024-11-17T08:14:19,313 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:14:19,313 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 988129a9cf65d53c5913e9c54b149fd7:A 2024-11-17T08:14:19,313 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T08:14:19,317 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T08:14:19,317 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): 988129a9cf65d53c5913e9c54b149fd7/C is initiating minor compaction (all files) 2024-11-17T08:14:19,317 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 988129a9cf65d53c5913e9c54b149fd7/C in TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:19,317 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/a49cb8bbdd9e4e0b8b29fc9e92722869, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/e0a24c7a91fe43dd88155f4f8c00f247, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/eb1463a46c434432aa1c943de9e6d849, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/c08eafd1f1c4414fbcf21d1e2d731e14] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp, totalSize=47.1 K 2024-11-17T08:14:19,318 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting a49cb8bbdd9e4e0b8b29fc9e92722869, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731831256109 2024-11-17T08:14:19,319 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0a24c7a91fe43dd88155f4f8c00f247, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731831257061 2024-11-17T08:14:19,320 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb1463a46c434432aa1c943de9e6d849, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731831257279 2024-11-17T08:14:19,321 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting c08eafd1f1c4414fbcf21d1e2d731e14, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731831257962 2024-11-17T08:14:19,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:19,332 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 988129a9cf65d53c5913e9c54b149fd7 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-17T08:14:19,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=A 2024-11-17T08:14:19,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:19,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=B 2024-11-17T08:14:19,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:19,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=C 2024-11-17T08:14:19,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:19,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/d8734eff0d584e1ba8e99cef0eb670ef is 50, key is test_row_0/A:col10/1731831258689/Put/seqid=0 2024-11-17T08:14:19,359 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 988129a9cf65d53c5913e9c54b149fd7#C#compaction#24 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:14:19,360 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/b101c1b0311b4c23af2dbf2095f9068b is 50, key is test_row_0/C:col10/1731831258614/Put/seqid=0 2024-11-17T08:14:19,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741860_1036 (size=14541) 2024-11-17T08:14:19,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741861_1037 (size=12391) 2024-11-17T08:14:19,404 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/b101c1b0311b4c23af2dbf2095f9068b as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/b101c1b0311b4c23af2dbf2095f9068b 2024-11-17T08:14:19,427 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 988129a9cf65d53c5913e9c54b149fd7/C of 988129a9cf65d53c5913e9c54b149fd7 into b101c1b0311b4c23af2dbf2095f9068b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:14:19,427 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:19,427 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., storeName=988129a9cf65d53c5913e9c54b149fd7/C, priority=12, startTime=1731831259183; duration=0sec 2024-11-17T08:14:19,428 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:19,428 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 988129a9cf65d53c5913e9c54b149fd7:C 2024-11-17T08:14:19,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T08:14:19,435 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:14:19,435 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:14:19,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-17T08:14:19,439 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:14:19,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T08:14:19,440 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:14:19,440 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:14:19,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:19,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831319433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:19,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:19,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:19,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:19,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:19,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831319446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:19,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:19,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:19,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:19,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:19,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831319448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:19,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:19,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:19,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:19,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831319450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:19,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:19,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:19,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:19,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:19,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:19,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831319453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:19,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:19,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:19,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:19,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T08:14:19,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:19,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831319554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:19,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:19,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:19,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:19,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:19,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831319563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:19,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:19,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:19,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:19,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:19,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831319563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:19,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:19,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:19,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:19,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:19,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831319563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:19,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:19,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831319564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:19,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:19,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:19,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:19,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:19,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:19,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:19,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-17T08:14:19,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:19,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:19,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:19,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:19,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:19,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:19,690 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/04955baeea824b67b1a78d23d398f3e1 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/04955baeea824b67b1a78d23d398f3e1 2024-11-17T08:14:19,702 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 988129a9cf65d53c5913e9c54b149fd7/B of 988129a9cf65d53c5913e9c54b149fd7 into 04955baeea824b67b1a78d23d398f3e1(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:14:19,702 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:19,702 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., storeName=988129a9cf65d53c5913e9c54b149fd7/B, priority=12, startTime=1731831259183; duration=0sec 2024-11-17T08:14:19,702 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:19,702 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 988129a9cf65d53c5913e9c54b149fd7:B 2024-11-17T08:14:19,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T08:14:19,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-17T08:14:19,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:19,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:19,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:19,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:19,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:19,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:19,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:19,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831319764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:19,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:19,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:19,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:19,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/d8734eff0d584e1ba8e99cef0eb670ef 2024-11-17T08:14:19,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:19,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831319775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:19,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:19,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831319775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:19,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:19,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:19,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:19,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:19,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831319775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:19,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:19,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:19,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:19,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:19,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:19,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:19,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:19,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831319784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:19,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:19,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:19,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:19,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/f6dd1472b9db42c3b34a5d0561cb3c44 is 50, key is test_row_0/B:col10/1731831258689/Put/seqid=0 2024-11-17T08:14:19,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741862_1038 (size=12151) 2024-11-17T08:14:19,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/f6dd1472b9db42c3b34a5d0561cb3c44 2024-11-17T08:14:19,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/aba29b187c8446ffb1522e68f69ccedc is 50, key is test_row_0/C:col10/1731831258689/Put/seqid=0 2024-11-17T08:14:19,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741863_1039 (size=12151) 2024-11-17T08:14:19,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/aba29b187c8446ffb1522e68f69ccedc 2024-11-17T08:14:19,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/d8734eff0d584e1ba8e99cef0eb670ef as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/d8734eff0d584e1ba8e99cef0eb670ef 2024-11-17T08:14:19,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/d8734eff0d584e1ba8e99cef0eb670ef, entries=200, sequenceid=157, filesize=14.2 K 2024-11-17T08:14:19,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/f6dd1472b9db42c3b34a5d0561cb3c44 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/f6dd1472b9db42c3b34a5d0561cb3c44 2024-11-17T08:14:19,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/f6dd1472b9db42c3b34a5d0561cb3c44, entries=150, sequenceid=157, filesize=11.9 K 2024-11-17T08:14:19,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/aba29b187c8446ffb1522e68f69ccedc as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/aba29b187c8446ffb1522e68f69ccedc 2024-11-17T08:14:19,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/aba29b187c8446ffb1522e68f69ccedc, entries=150, sequenceid=157, filesize=11.9 K 2024-11-17T08:14:19,895 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 988129a9cf65d53c5913e9c54b149fd7 in 563ms, sequenceid=157, compaction requested=false 2024-11-17T08:14:19,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:19,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-17T08:14:19,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:19,910 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 988129a9cf65d53c5913e9c54b149fd7 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-17T08:14:19,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=A 2024-11-17T08:14:19,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:19,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=B 2024-11-17T08:14:19,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:19,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=C 2024-11-17T08:14:19,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:19,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/d5a8ec86e5c24d548f224cb22c1e5dad is 50, key is test_row_0/A:col10/1731831259450/Put/seqid=0 2024-11-17T08:14:19,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741864_1040 (size=12151) 2024-11-17T08:14:19,943 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/d5a8ec86e5c24d548f224cb22c1e5dad 2024-11-17T08:14:19,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/a2d0c489105b45df93ea71a2cf90e99f is 50, key is test_row_0/B:col10/1731831259450/Put/seqid=0 2024-11-17T08:14:19,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741865_1041 (size=12151) 2024-11-17T08:14:19,996 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/a2d0c489105b45df93ea71a2cf90e99f 2024-11-17T08:14:20,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/67e9f90ec02c47cdac83666f3f209be1 is 50, key is test_row_0/C:col10/1731831259450/Put/seqid=0 2024-11-17T08:14:20,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741866_1042 (size=12151) 2024-11-17T08:14:20,040 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/67e9f90ec02c47cdac83666f3f209be1 2024-11-17T08:14:20,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/d5a8ec86e5c24d548f224cb22c1e5dad as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/d5a8ec86e5c24d548f224cb22c1e5dad 2024-11-17T08:14:20,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T08:14:20,071 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/d5a8ec86e5c24d548f224cb22c1e5dad, entries=150, sequenceid=173, filesize=11.9 K 2024-11-17T08:14:20,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/a2d0c489105b45df93ea71a2cf90e99f as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/a2d0c489105b45df93ea71a2cf90e99f 2024-11-17T08:14:20,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:20,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:20,089 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/a2d0c489105b45df93ea71a2cf90e99f, entries=150, sequenceid=173, filesize=11.9 K 2024-11-17T08:14:20,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/67e9f90ec02c47cdac83666f3f209be1 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/67e9f90ec02c47cdac83666f3f209be1 2024-11-17T08:14:20,102 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/67e9f90ec02c47cdac83666f3f209be1, entries=150, sequenceid=173, filesize=11.9 K 2024-11-17T08:14:20,104 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=87.22 KB/89310 for 988129a9cf65d53c5913e9c54b149fd7 in 193ms, sequenceid=173, compaction requested=true 2024-11-17T08:14:20,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:20,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:20,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-17T08:14:20,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-17T08:14:20,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:20,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 988129a9cf65d53c5913e9c54b149fd7 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-17T08:14:20,109 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-17T08:14:20,109 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 666 msec 2024-11-17T08:14:20,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=A 2024-11-17T08:14:20,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:20,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=B 2024-11-17T08:14:20,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:20,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=C 2024-11-17T08:14:20,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:20,115 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 676 msec 2024-11-17T08:14:20,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/649f289786f1491c9f7c6e068a2bd3d3 is 50, key is test_row_0/A:col10/1731831260105/Put/seqid=0 2024-11-17T08:14:20,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:20,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:20,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831320124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:20,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831320128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:20,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:20,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:20,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:20,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741867_1043 (size=12151) 2024-11-17T08:14:20,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831320136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:20,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:20,144 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/649f289786f1491c9f7c6e068a2bd3d3 2024-11-17T08:14:20,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:20,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831320138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:20,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:20,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:20,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831320138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:20,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:20,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/435f65d529cf44cfbd90396f92080f23 is 50, key is test_row_0/B:col10/1731831260105/Put/seqid=0 2024-11-17T08:14:20,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741868_1044 (size=12151) 2024-11-17T08:14:20,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/435f65d529cf44cfbd90396f92080f23 2024-11-17T08:14:20,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/7c9ef0b184a143f3ae33e85acc91b349 is 50, key is test_row_0/C:col10/1731831260105/Put/seqid=0 2024-11-17T08:14:20,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741869_1045 (size=12151) 2024-11-17T08:14:20,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:20,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831320244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:20,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:20,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:20,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831320244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:20,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:20,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:20,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831320253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:20,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:20,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:20,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831320253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:20,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:20,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:20,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831320253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:20,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:20,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:20,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831320454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:20,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:20,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:20,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831320454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:20,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:20,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:20,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831320464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:20,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:20,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831320464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:20,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:20,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831320464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:20,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:20,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:20,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:20,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T08:14:20,574 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:14:20,575 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:14:20,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-17T08:14:20,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T08:14:20,578 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:14:20,580 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:14:20,580 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:14:20,620 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/7c9ef0b184a143f3ae33e85acc91b349 2024-11-17T08:14:20,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/649f289786f1491c9f7c6e068a2bd3d3 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/649f289786f1491c9f7c6e068a2bd3d3 2024-11-17T08:14:20,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/649f289786f1491c9f7c6e068a2bd3d3, entries=150, sequenceid=191, filesize=11.9 K 2024-11-17T08:14:20,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/435f65d529cf44cfbd90396f92080f23 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/435f65d529cf44cfbd90396f92080f23 2024-11-17T08:14:20,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/435f65d529cf44cfbd90396f92080f23, entries=150, sequenceid=191, filesize=11.9 K 2024-11-17T08:14:20,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/7c9ef0b184a143f3ae33e85acc91b349 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/7c9ef0b184a143f3ae33e85acc91b349 2024-11-17T08:14:20,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/7c9ef0b184a143f3ae33e85acc91b349, entries=150, sequenceid=191, filesize=11.9 K 2024-11-17T08:14:20,670 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 988129a9cf65d53c5913e9c54b149fd7 in 562ms, sequenceid=191, compaction requested=true 2024-11-17T08:14:20,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:20,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 988129a9cf65d53c5913e9c54b149fd7:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:14:20,671 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T08:14:20,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:20,671 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T08:14:20,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 988129a9cf65d53c5913e9c54b149fd7:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:14:20,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:20,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 988129a9cf65d53c5913e9c54b149fd7:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:14:20,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:14:20,673 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T08:14:20,673 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51234 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T08:14:20,673 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): 988129a9cf65d53c5913e9c54b149fd7/B is initiating minor compaction (all files) 2024-11-17T08:14:20,673 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): 988129a9cf65d53c5913e9c54b149fd7/A is initiating minor compaction (all files) 2024-11-17T08:14:20,673 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 988129a9cf65d53c5913e9c54b149fd7/A in TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:20,673 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 988129a9cf65d53c5913e9c54b149fd7/B in TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:20,673 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/975ccd3742eb429ca765486bfc976e87, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/d8734eff0d584e1ba8e99cef0eb670ef, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/d5a8ec86e5c24d548f224cb22c1e5dad, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/649f289786f1491c9f7c6e068a2bd3d3] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp, totalSize=50.0 K 2024-11-17T08:14:20,673 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/04955baeea824b67b1a78d23d398f3e1, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/f6dd1472b9db42c3b34a5d0561cb3c44, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/a2d0c489105b45df93ea71a2cf90e99f, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/435f65d529cf44cfbd90396f92080f23] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp, totalSize=47.7 K 2024-11-17T08:14:20,674 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 04955baeea824b67b1a78d23d398f3e1, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731831257962 2024-11-17T08:14:20,674 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 975ccd3742eb429ca765486bfc976e87, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731831257962 2024-11-17T08:14:20,675 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8734eff0d584e1ba8e99cef0eb670ef, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731831258689 2024-11-17T08:14:20,675 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting f6dd1472b9db42c3b34a5d0561cb3c44, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731831258689 2024-11-17T08:14:20,675 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5a8ec86e5c24d548f224cb22c1e5dad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731831259428 2024-11-17T08:14:20,676 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 649f289786f1491c9f7c6e068a2bd3d3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731831260096 2024-11-17T08:14:20,676 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting a2d0c489105b45df93ea71a2cf90e99f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731831259428 2024-11-17T08:14:20,677 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 435f65d529cf44cfbd90396f92080f23, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731831260096 2024-11-17T08:14:20,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T08:14:20,698 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 988129a9cf65d53c5913e9c54b149fd7#B#compaction#33 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:14:20,698 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/184b44b7f6934b6eb3fc1212b88ad8c7 is 50, key is test_row_0/B:col10/1731831260105/Put/seqid=0 2024-11-17T08:14:20,707 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 988129a9cf65d53c5913e9c54b149fd7#A#compaction#34 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:14:20,708 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/4ceffb75b37f4d0d98ba4c0d154eb574 is 50, key is test_row_0/A:col10/1731831260105/Put/seqid=0 2024-11-17T08:14:20,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-17T08:14:20,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741871_1047 (size=12527) 2024-11-17T08:14:20,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:20,734 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 988129a9cf65d53c5913e9c54b149fd7 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-17T08:14:20,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=A 2024-11-17T08:14:20,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:20,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=B 2024-11-17T08:14:20,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:20,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=C 2024-11-17T08:14:20,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:20,752 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/4ceffb75b37f4d0d98ba4c0d154eb574 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/4ceffb75b37f4d0d98ba4c0d154eb574 2024-11-17T08:14:20,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741870_1046 (size=12527) 2024-11-17T08:14:20,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/6f1a31bf29884c46a0786d5b15ae25f2 is 50, key is test_row_0/A:col10/1731831260132/Put/seqid=0 2024-11-17T08:14:20,762 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 988129a9cf65d53c5913e9c54b149fd7/A of 988129a9cf65d53c5913e9c54b149fd7 into 4ceffb75b37f4d0d98ba4c0d154eb574(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:14:20,763 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:20,763 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., storeName=988129a9cf65d53c5913e9c54b149fd7/A, priority=12, startTime=1731831260671; duration=0sec 2024-11-17T08:14:20,763 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:14:20,763 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 988129a9cf65d53c5913e9c54b149fd7:A 2024-11-17T08:14:20,763 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T08:14:20,769 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T08:14:20,769 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): 988129a9cf65d53c5913e9c54b149fd7/C is initiating minor compaction (all files) 2024-11-17T08:14:20,770 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 988129a9cf65d53c5913e9c54b149fd7/C in TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:20,770 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/b101c1b0311b4c23af2dbf2095f9068b, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/aba29b187c8446ffb1522e68f69ccedc, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/67e9f90ec02c47cdac83666f3f209be1, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/7c9ef0b184a143f3ae33e85acc91b349] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp, totalSize=47.7 K 2024-11-17T08:14:20,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:20,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:20,771 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting b101c1b0311b4c23af2dbf2095f9068b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731831257962 2024-11-17T08:14:20,772 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting aba29b187c8446ffb1522e68f69ccedc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731831258689 2024-11-17T08:14:20,773 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/184b44b7f6934b6eb3fc1212b88ad8c7 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/184b44b7f6934b6eb3fc1212b88ad8c7 2024-11-17T08:14:20,775 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67e9f90ec02c47cdac83666f3f209be1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731831259428 2024-11-17T08:14:20,775 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c9ef0b184a143f3ae33e85acc91b349, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731831260096 2024-11-17T08:14:20,791 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 988129a9cf65d53c5913e9c54b149fd7/B of 988129a9cf65d53c5913e9c54b149fd7 into 184b44b7f6934b6eb3fc1212b88ad8c7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:14:20,792 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:20,792 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., storeName=988129a9cf65d53c5913e9c54b149fd7/B, priority=12, startTime=1731831260671; duration=0sec 2024-11-17T08:14:20,793 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:20,793 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 988129a9cf65d53c5913e9c54b149fd7:B 2024-11-17T08:14:20,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741872_1048 (size=12151) 2024-11-17T08:14:20,803 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/6f1a31bf29884c46a0786d5b15ae25f2 2024-11-17T08:14:20,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:20,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831320797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:20,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:20,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:20,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831320797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:20,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:20,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:20,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831320801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:20,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:20,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:20,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831320799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:20,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,814 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 988129a9cf65d53c5913e9c54b149fd7#C#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:14:20,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:20,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:20,815 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/0fd27848b911494190352cee1654b127 is 50, key is test_row_0/C:col10/1731831260105/Put/seqid=0 2024-11-17T08:14:20,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831320805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:20,816 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,816 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,816 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:20,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/d0e90d4700e24a2c9d0899b614c6a021 is 50, key is test_row_0/B:col10/1731831260132/Put/seqid=0 2024-11-17T08:14:20,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741873_1049 (size=12527) 2024-11-17T08:14:20,868 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/0fd27848b911494190352cee1654b127 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/0fd27848b911494190352cee1654b127 2024-11-17T08:14:20,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741874_1050 (size=12151) 2024-11-17T08:14:20,876 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/d0e90d4700e24a2c9d0899b614c6a021 2024-11-17T08:14:20,886 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 988129a9cf65d53c5913e9c54b149fd7/C of 988129a9cf65d53c5913e9c54b149fd7 into 0fd27848b911494190352cee1654b127(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:14:20,886 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:20,886 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., storeName=988129a9cf65d53c5913e9c54b149fd7/C, priority=12, startTime=1731831260671; duration=0sec 2024-11-17T08:14:20,887 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:20,887 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 988129a9cf65d53c5913e9c54b149fd7:C 2024-11-17T08:14:20,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/0f83f3225d77477088d9edca01cf4a20 is 50, key is test_row_0/C:col10/1731831260132/Put/seqid=0 2024-11-17T08:14:20,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T08:14:20,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741875_1051 (size=12151) 2024-11-17T08:14:20,915 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/0f83f3225d77477088d9edca01cf4a20 2024-11-17T08:14:20,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:20,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831320914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:20,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:20,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:20,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831320915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:20,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:20,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:20,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831320924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:20,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:20,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:20,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831320924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:20,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:20,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831320925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:20,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:20,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:20,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:20,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/6f1a31bf29884c46a0786d5b15ae25f2 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/6f1a31bf29884c46a0786d5b15ae25f2 2024-11-17T08:14:20,941 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/6f1a31bf29884c46a0786d5b15ae25f2, entries=150, sequenceid=206, filesize=11.9 K 2024-11-17T08:14:20,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/d0e90d4700e24a2c9d0899b614c6a021 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/d0e90d4700e24a2c9d0899b614c6a021 2024-11-17T08:14:20,955 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/d0e90d4700e24a2c9d0899b614c6a021, entries=150, sequenceid=206, filesize=11.9 K 2024-11-17T08:14:20,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/0f83f3225d77477088d9edca01cf4a20 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/0f83f3225d77477088d9edca01cf4a20 2024-11-17T08:14:20,966 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/0f83f3225d77477088d9edca01cf4a20, entries=150, sequenceid=206, filesize=11.9 K 2024-11-17T08:14:20,968 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 988129a9cf65d53c5913e9c54b149fd7 in 233ms, sequenceid=206, compaction requested=false 2024-11-17T08:14:20,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:20,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:20,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-17T08:14:20,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-17T08:14:20,974 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-17T08:14:20,975 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 390 msec 2024-11-17T08:14:20,979 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 400 msec 2024-11-17T08:14:21,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:21,137 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 988129a9cf65d53c5913e9c54b149fd7 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-17T08:14:21,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=A 2024-11-17T08:14:21,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:21,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=B 2024-11-17T08:14:21,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:21,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=C 2024-11-17T08:14:21,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:21,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/3bd90273cca0436d8154c5d476b4bcab is 50, key is test_row_0/A:col10/1731831260801/Put/seqid=0 2024-11-17T08:14:21,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:21,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831321149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:21,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:21,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:21,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:21,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831321151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:21,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:21,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:21,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:21,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:21,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:21,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831321151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:21,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:21,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831321155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:21,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:21,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:21,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:21,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:21,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:21,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:21,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:21,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831321158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:21,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:21,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:21,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:21,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741876_1052 (size=16931) 2024-11-17T08:14:21,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T08:14:21,203 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:14:21,204 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:14:21,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-17T08:14:21,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T08:14:21,207 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:14:21,208 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:14:21,208 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:14:21,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:21,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831321264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:21,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:21,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:21,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:21,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:21,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831321264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:21,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:21,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831321264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:21,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:21,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:21,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:21,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:21,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831321264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:21,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:21,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:21,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:21,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:21,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:21,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:21,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:21,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831321274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:21,275 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:21,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:21,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:21,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T08:14:21,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T08:14:21,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:21,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:21,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:21,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:21,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:21,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:21,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:21,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831321474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:21,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:21,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:21,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:21,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:21,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831321474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:21,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:21,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831321475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:21,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:21,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:21,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:21,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:21,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:21,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:21,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:21,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831321475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:21,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:21,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:21,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:21,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:21,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831321484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:21,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:21,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:21,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:21,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T08:14:21,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:21,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:21,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:21,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:21,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:21,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T08:14:21,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:21,575 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/3bd90273cca0436d8154c5d476b4bcab 2024-11-17T08:14:21,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/1c28f33dc48043d2b97a8a0b9a20b594 is 50, key is test_row_0/B:col10/1731831260801/Put/seqid=0 2024-11-17T08:14:21,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741877_1053 (size=12151) 2024-11-17T08:14:21,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T08:14:21,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:21,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:21,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:21,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:21,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:21,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:21,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:21,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:21,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831321784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:21,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831321784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:21,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:21,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:21,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:21,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:21,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:21,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:21,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:21,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831321785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:21,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:21,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:21,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:21,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:21,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831321785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:21,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:21,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:21,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:21,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:21,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831321793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:21,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:21,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:21,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:21,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T08:14:21,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:21,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:21,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:21,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:21,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:21,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:21,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T08:14:21,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T08:14:21,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:21,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:21,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:21,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:21,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:21,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:22,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/1c28f33dc48043d2b97a8a0b9a20b594 2024-11-17T08:14:22,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/f2d1510f85ec465483fe6ed04726f37f is 50, key is test_row_0/C:col10/1731831260801/Put/seqid=0 2024-11-17T08:14:22,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741878_1054 (size=12151) 2024-11-17T08:14:22,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/f2d1510f85ec465483fe6ed04726f37f 2024-11-17T08:14:22,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/3bd90273cca0436d8154c5d476b4bcab as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/3bd90273cca0436d8154c5d476b4bcab 2024-11-17T08:14:22,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/3bd90273cca0436d8154c5d476b4bcab, entries=250, sequenceid=228, filesize=16.5 K 2024-11-17T08:14:22,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/1c28f33dc48043d2b97a8a0b9a20b594 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/1c28f33dc48043d2b97a8a0b9a20b594 2024-11-17T08:14:22,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/1c28f33dc48043d2b97a8a0b9a20b594, entries=150, sequenceid=228, filesize=11.9 K 2024-11-17T08:14:22,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/f2d1510f85ec465483fe6ed04726f37f as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/f2d1510f85ec465483fe6ed04726f37f 2024-11-17T08:14:22,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/f2d1510f85ec465483fe6ed04726f37f, entries=150, sequenceid=228, filesize=11.9 K 2024-11-17T08:14:22,085 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 988129a9cf65d53c5913e9c54b149fd7 in 948ms, sequenceid=228, compaction requested=true 2024-11-17T08:14:22,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:22,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 988129a9cf65d53c5913e9c54b149fd7:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:14:22,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:22,087 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:14:22,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 988129a9cf65d53c5913e9c54b149fd7:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:14:22,087 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:14:22,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:22,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 988129a9cf65d53c5913e9c54b149fd7:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:14:22,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:14:22,088 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:14:22,088 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41609 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:14:22,088 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): 988129a9cf65d53c5913e9c54b149fd7/B is initiating minor compaction (all files) 2024-11-17T08:14:22,088 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): 988129a9cf65d53c5913e9c54b149fd7/A is initiating minor compaction (all files) 2024-11-17T08:14:22,088 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 988129a9cf65d53c5913e9c54b149fd7/B in TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:22,088 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 988129a9cf65d53c5913e9c54b149fd7/A in TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:22,088 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/184b44b7f6934b6eb3fc1212b88ad8c7, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/d0e90d4700e24a2c9d0899b614c6a021, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/1c28f33dc48043d2b97a8a0b9a20b594] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp, totalSize=36.0 K 2024-11-17T08:14:22,088 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/4ceffb75b37f4d0d98ba4c0d154eb574, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/6f1a31bf29884c46a0786d5b15ae25f2, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/3bd90273cca0436d8154c5d476b4bcab] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp, totalSize=40.6 K 2024-11-17T08:14:22,089 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 184b44b7f6934b6eb3fc1212b88ad8c7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731831260096 2024-11-17T08:14:22,089 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ceffb75b37f4d0d98ba4c0d154eb574, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731831260096 2024-11-17T08:14:22,090 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting d0e90d4700e24a2c9d0899b614c6a021, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731831260132 2024-11-17T08:14:22,090 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f1a31bf29884c46a0786d5b15ae25f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731831260132 2024-11-17T08:14:22,092 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c28f33dc48043d2b97a8a0b9a20b594, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731831260792 2024-11-17T08:14:22,092 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bd90273cca0436d8154c5d476b4bcab, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731831260792 2024-11-17T08:14:22,113 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 988129a9cf65d53c5913e9c54b149fd7#A#compaction#42 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:14:22,114 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 988129a9cf65d53c5913e9c54b149fd7#B#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:14:22,114 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/49abbb97a36942498593210efe3404e4 is 50, key is test_row_0/A:col10/1731831260801/Put/seqid=0 2024-11-17T08:14:22,114 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/6b0ef1df3f754883b888a6995f4cdd12 is 50, key is test_row_0/B:col10/1731831260801/Put/seqid=0 2024-11-17T08:14:22,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741879_1055 (size=12629) 2024-11-17T08:14:22,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741880_1056 (size=12629) 2024-11-17T08:14:22,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T08:14:22,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:22,143 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 988129a9cf65d53c5913e9c54b149fd7 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-17T08:14:22,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=A 2024-11-17T08:14:22,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:22,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=B 2024-11-17T08:14:22,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:22,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=C 2024-11-17T08:14:22,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:22,156 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/49abbb97a36942498593210efe3404e4 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/49abbb97a36942498593210efe3404e4 2024-11-17T08:14:22,165 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/6b0ef1df3f754883b888a6995f4cdd12 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/6b0ef1df3f754883b888a6995f4cdd12 2024-11-17T08:14:22,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/fb6dc5ab09a94e5a83012f69da13a7e7 is 50, key is test_row_0/A:col10/1731831261148/Put/seqid=0 2024-11-17T08:14:22,177 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 988129a9cf65d53c5913e9c54b149fd7/B of 988129a9cf65d53c5913e9c54b149fd7 into 6b0ef1df3f754883b888a6995f4cdd12(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:14:22,178 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:22,178 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., storeName=988129a9cf65d53c5913e9c54b149fd7/B, priority=13, startTime=1731831262087; duration=0sec 2024-11-17T08:14:22,178 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:14:22,178 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 988129a9cf65d53c5913e9c54b149fd7:B 2024-11-17T08:14:22,178 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:14:22,182 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:14:22,182 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 988129a9cf65d53c5913e9c54b149fd7/A of 988129a9cf65d53c5913e9c54b149fd7 into 49abbb97a36942498593210efe3404e4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:14:22,182 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:22,182 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): 988129a9cf65d53c5913e9c54b149fd7/C is initiating minor compaction (all files) 2024-11-17T08:14:22,182 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., storeName=988129a9cf65d53c5913e9c54b149fd7/A, priority=13, startTime=1731831262086; duration=0sec 2024-11-17T08:14:22,182 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 988129a9cf65d53c5913e9c54b149fd7/C in TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:22,182 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/0fd27848b911494190352cee1654b127, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/0f83f3225d77477088d9edca01cf4a20, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/f2d1510f85ec465483fe6ed04726f37f] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp, totalSize=36.0 K 2024-11-17T08:14:22,182 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:22,182 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 988129a9cf65d53c5913e9c54b149fd7:A 2024-11-17T08:14:22,183 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fd27848b911494190352cee1654b127, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731831260096 2024-11-17T08:14:22,183 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f83f3225d77477088d9edca01cf4a20, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731831260132 2024-11-17T08:14:22,184 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting f2d1510f85ec465483fe6ed04726f37f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731831260792 2024-11-17T08:14:22,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741881_1057 (size=12151) 2024-11-17T08:14:22,202 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 988129a9cf65d53c5913e9c54b149fd7#C#compaction#45 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:14:22,204 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/13f2bbbc40cc47b6972ed2ff16ace1f2 is 50, key is test_row_0/C:col10/1731831260801/Put/seqid=0 2024-11-17T08:14:22,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741882_1058 (size=12629) 2024-11-17T08:14:22,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:22,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:22,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:22,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831322315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:22,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:22,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:22,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:22,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:22,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831322317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:22,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:22,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:22,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:22,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:22,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831322319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:22,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:22,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:22,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:22,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:22,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831322321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:22,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:22,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:22,330 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:22,330 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:22,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831322323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:22,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:22,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:22,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:22,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T08:14:22,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831322433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:22,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:22,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:22,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831322434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:22,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:22,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:22,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:22,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:22,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:22,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831322434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:22,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:22,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:22,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:22,436 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:22,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831322434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:22,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:22,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:22,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:22,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:22,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831322434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:22,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:22,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:22,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:22,598 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/fb6dc5ab09a94e5a83012f69da13a7e7 2024-11-17T08:14:22,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/c803d88c0238470a80618a4a3fe7d049 is 50, key is test_row_0/B:col10/1731831261148/Put/seqid=0 2024-11-17T08:14:22,632 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/13f2bbbc40cc47b6972ed2ff16ace1f2 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/13f2bbbc40cc47b6972ed2ff16ace1f2 2024-11-17T08:14:22,644 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 988129a9cf65d53c5913e9c54b149fd7/C of 988129a9cf65d53c5913e9c54b149fd7 into 13f2bbbc40cc47b6972ed2ff16ace1f2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:14:22,644 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:22,644 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., storeName=988129a9cf65d53c5913e9c54b149fd7/C, priority=13, startTime=1731831262087; duration=0sec 2024-11-17T08:14:22,644 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:22,644 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 988129a9cf65d53c5913e9c54b149fd7:C 2024-11-17T08:14:22,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:22,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831322644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:22,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:22,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:22,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:22,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831322645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:22,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:22,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:22,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:22,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831322645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:22,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:22,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:22,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831322645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:22,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:22,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741883_1059 (size=12151) 2024-11-17T08:14:22,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:22,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:22,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:22,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:22,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:22,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831322645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:22,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:22,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:22,651 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/c803d88c0238470a80618a4a3fe7d049 2024-11-17T08:14:22,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:22,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/822f2d1db44c40e68d6d85eea454e026 is 50, key is test_row_0/C:col10/1731831261148/Put/seqid=0 2024-11-17T08:14:22,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741884_1060 (size=12151) 2024-11-17T08:14:22,690 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/822f2d1db44c40e68d6d85eea454e026 2024-11-17T08:14:22,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/fb6dc5ab09a94e5a83012f69da13a7e7 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/fb6dc5ab09a94e5a83012f69da13a7e7 2024-11-17T08:14:22,713 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/fb6dc5ab09a94e5a83012f69da13a7e7, entries=150, sequenceid=241, filesize=11.9 K 2024-11-17T08:14:22,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/c803d88c0238470a80618a4a3fe7d049 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/c803d88c0238470a80618a4a3fe7d049 2024-11-17T08:14:22,721 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/c803d88c0238470a80618a4a3fe7d049, entries=150, sequenceid=241, filesize=11.9 K 2024-11-17T08:14:22,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/822f2d1db44c40e68d6d85eea454e026 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/822f2d1db44c40e68d6d85eea454e026 2024-11-17T08:14:22,728 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/822f2d1db44c40e68d6d85eea454e026, entries=150, sequenceid=241, filesize=11.9 K 2024-11-17T08:14:22,746 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 988129a9cf65d53c5913e9c54b149fd7 in 603ms, sequenceid=241, compaction requested=false 2024-11-17T08:14:22,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:22,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:22,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-17T08:14:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-17T08:14:22,755 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-17T08:14:22,755 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5420 sec 2024-11-17T08:14:22,760 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.5520 sec 2024-11-17T08:14:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:22,958 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731831262313, max timestamp=1731831262955Num uniques 859; 2024-11-17T08:14:22,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T08:14:22,960 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731831262313, max timestamp=1731831262955Num uniques 859; 2024-11-17T08:14:22,960 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T08:14:22,961 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 988129a9cf65d53c5913e9c54b149fd7 3/3 column families, dataSize=76.04 KB heapSize=214.17 KB 2024-11-17T08:14:22,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=A 2024-11-17T08:14:22,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:14:22,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=B 2024-11-17T08:14:22,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:14:22,960 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731831262313, max timestamp=1731831262955Num uniques 859; 2024-11-17T08:14:22,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=C 2024-11-17T08:14:22,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:14:22,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T08:14:22,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/0c5b18921bf24f0e9ad99aabc802dc28 is 50, key is test_row_0/A:col10/1731831262957/Put/seqid=0 2024-11-17T08:14:22,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741885_1061 (size=17181) 2024-11-17T08:14:23,039 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731831263019, max timestamp=1731831263036Num uniques 859; 2024-11-17T08:14:23,039 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T08:14:23,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731831263019, max timestamp=1731831263036Num uniques 859; 2024-11-17T08:14:23,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T08:14:23,041 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731831263019, max timestamp=1731831263036Num uniques 859; 2024-11-17T08:14:23,041 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-17T08:14:23,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:23,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831323065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:23,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:23,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831323067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:23,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:23,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831323067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:23,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:23,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:23,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831323068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:23,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:23,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:23,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831323070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:23,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:23,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:23,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:23,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831323183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831323183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:23,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:23,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:23,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831323183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831323184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:23,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:23,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:23,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:23,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:23,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831323184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:23,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:23,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T08:14:23,354 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:14:23,355 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:14:23,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-17T08:14:23,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T08:14:23,358 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:14:23,359 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:14:23,359 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:14:23,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/0c5b18921bf24f0e9ad99aabc802dc28 2024-11-17T08:14:23,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/4a80046370f943398f32f26e2eb8c388 is 50, key is test_row_0/B:col10/1731831262957/Put/seqid=0 2024-11-17T08:14:23,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:23,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:23,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831323404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:23,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831323404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:23,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:23,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:23,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:23,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:23,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831323405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:23,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831323405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:23,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:23,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831323405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:23,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:23,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:23,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:23,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741886_1062 (size=12301) 2024-11-17T08:14:23,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/4a80046370f943398f32f26e2eb8c388 2024-11-17T08:14:23,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/fbb72ae625fa41b29b461b10c6d905ce is 50, key is test_row_0/C:col10/1731831262957/Put/seqid=0 2024-11-17T08:14:23,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741887_1063 (size=12301) 2024-11-17T08:14:23,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/fbb72ae625fa41b29b461b10c6d905ce 2024-11-17T08:14:23,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T08:14:23,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/0c5b18921bf24f0e9ad99aabc802dc28 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/0c5b18921bf24f0e9ad99aabc802dc28 2024-11-17T08:14:23,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/0c5b18921bf24f0e9ad99aabc802dc28, entries=250, sequenceid=267, filesize=16.8 K 2024-11-17T08:14:23,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/4a80046370f943398f32f26e2eb8c388 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/4a80046370f943398f32f26e2eb8c388 2024-11-17T08:14:23,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/4a80046370f943398f32f26e2eb8c388, entries=150, sequenceid=267, filesize=12.0 K 2024-11-17T08:14:23,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/fbb72ae625fa41b29b461b10c6d905ce as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/fbb72ae625fa41b29b461b10c6d905ce 2024-11-17T08:14:23,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/fbb72ae625fa41b29b461b10c6d905ce, entries=150, sequenceid=267, filesize=12.0 K 2024-11-17T08:14:23,496 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~121.27 KB/124176, currentSize=142.73 KB/146155 for 988129a9cf65d53c5913e9c54b149fd7 in 536ms, sequenceid=267, compaction requested=true 2024-11-17T08:14:23,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:23,496 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:14:23,498 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41961 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:14:23,498 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): 988129a9cf65d53c5913e9c54b149fd7/A is initiating minor compaction (all files) 2024-11-17T08:14:23,498 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 988129a9cf65d53c5913e9c54b149fd7/A in TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:23,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 988129a9cf65d53c5913e9c54b149fd7:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:14:23,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:23,499 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/49abbb97a36942498593210efe3404e4, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/fb6dc5ab09a94e5a83012f69da13a7e7, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/0c5b18921bf24f0e9ad99aabc802dc28] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp, totalSize=41.0 K 2024-11-17T08:14:23,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 988129a9cf65d53c5913e9c54b149fd7:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:14:23,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:23,499 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:14:23,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 988129a9cf65d53c5913e9c54b149fd7:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:14:23,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:14:23,499 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49abbb97a36942498593210efe3404e4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731831260792 2024-11-17T08:14:23,500 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb6dc5ab09a94e5a83012f69da13a7e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731831261143 2024-11-17T08:14:23,501 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:14:23,501 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c5b18921bf24f0e9ad99aabc802dc28, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731831262316 2024-11-17T08:14:23,501 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): 988129a9cf65d53c5913e9c54b149fd7/B is initiating minor compaction (all files) 2024-11-17T08:14:23,501 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 988129a9cf65d53c5913e9c54b149fd7/B in TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:23,501 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/6b0ef1df3f754883b888a6995f4cdd12, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/c803d88c0238470a80618a4a3fe7d049, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/4a80046370f943398f32f26e2eb8c388] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp, totalSize=36.2 K 2024-11-17T08:14:23,502 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b0ef1df3f754883b888a6995f4cdd12, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731831260792 2024-11-17T08:14:23,503 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting c803d88c0238470a80618a4a3fe7d049, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731831261143 2024-11-17T08:14:23,505 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a80046370f943398f32f26e2eb8c388, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731831262955 2024-11-17T08:14:23,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-17T08:14:23,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:23,513 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 988129a9cf65d53c5913e9c54b149fd7 3/3 column families, dataSize=142.73 KB heapSize=404.38 KB 2024-11-17T08:14:23,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=A 2024-11-17T08:14:23,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:14:23,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=B 2024-11-17T08:14:23,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:14:23,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=C 2024-11-17T08:14:23,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:14:23,527 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 988129a9cf65d53c5913e9c54b149fd7#A#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:14:23,528 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/2bfb826ce23c49688e6192eac7223278 is 50, key is test_row_0/A:col10/1731831262957/Put/seqid=0 2024-11-17T08:14:23,531 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 988129a9cf65d53c5913e9c54b149fd7#B#compaction#53 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:14:23,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/0453524bc9194b1c9f0efbad6890ed0a is 50, key is test_row_0/A:col10/1731831263066/Put/seqid=0 2024-11-17T08:14:23,531 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/89e4c1af56b14cb79c365416a58276d2 is 50, key is test_row_0/B:col10/1731831262957/Put/seqid=0 2024-11-17T08:14:23,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741888_1064 (size=12881) 2024-11-17T08:14:23,571 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/2bfb826ce23c49688e6192eac7223278 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/2bfb826ce23c49688e6192eac7223278 2024-11-17T08:14:23,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741889_1065 (size=12301) 2024-11-17T08:14:23,576 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/0453524bc9194b1c9f0efbad6890ed0a 2024-11-17T08:14:23,588 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 988129a9cf65d53c5913e9c54b149fd7/A of 988129a9cf65d53c5913e9c54b149fd7 into 2bfb826ce23c49688e6192eac7223278(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:14:23,589 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:23,589 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., storeName=988129a9cf65d53c5913e9c54b149fd7/A, priority=13, startTime=1731831263496; duration=0sec 2024-11-17T08:14:23,589 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:14:23,589 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 988129a9cf65d53c5913e9c54b149fd7:A 2024-11-17T08:14:23,589 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:14:23,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741890_1066 (size=12881) 2024-11-17T08:14:23,596 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:14:23,596 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): 988129a9cf65d53c5913e9c54b149fd7/C is initiating minor compaction (all files) 2024-11-17T08:14:23,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/2aa124c48f9a4cdb91cf1f3c04cc0c3b is 50, key is test_row_0/B:col10/1731831263066/Put/seqid=0 2024-11-17T08:14:23,602 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 988129a9cf65d53c5913e9c54b149fd7/C in TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:23,604 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/13f2bbbc40cc47b6972ed2ff16ace1f2, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/822f2d1db44c40e68d6d85eea454e026, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/fbb72ae625fa41b29b461b10c6d905ce] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp, totalSize=36.2 K 2024-11-17T08:14:23,605 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13f2bbbc40cc47b6972ed2ff16ace1f2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731831260792 2024-11-17T08:14:23,606 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 822f2d1db44c40e68d6d85eea454e026, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731831261143 2024-11-17T08:14:23,606 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbb72ae625fa41b29b461b10c6d905ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731831262955 2024-11-17T08:14:23,611 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/89e4c1af56b14cb79c365416a58276d2 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/89e4c1af56b14cb79c365416a58276d2 2024-11-17T08:14:23,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741891_1067 (size=12301) 2024-11-17T08:14:23,625 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/2aa124c48f9a4cdb91cf1f3c04cc0c3b 2024-11-17T08:14:23,625 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 988129a9cf65d53c5913e9c54b149fd7/B of 988129a9cf65d53c5913e9c54b149fd7 into 89e4c1af56b14cb79c365416a58276d2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:14:23,626 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:23,626 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., storeName=988129a9cf65d53c5913e9c54b149fd7/B, priority=13, startTime=1731831263499; duration=0sec 2024-11-17T08:14:23,626 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:23,626 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 988129a9cf65d53c5913e9c54b149fd7:B 2024-11-17T08:14:23,630 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 988129a9cf65d53c5913e9c54b149fd7#C#compaction#55 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:14:23,630 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/7503fe94f84243b1950c7a5d864b3359 is 50, key is test_row_0/C:col10/1731831262957/Put/seqid=0 2024-11-17T08:14:23,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/bcfea8a65043478f876960630da2132a is 50, key is test_row_0/C:col10/1731831263066/Put/seqid=0 2024-11-17T08:14:23,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741892_1068 (size=12881) 2024-11-17T08:14:23,662 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/7503fe94f84243b1950c7a5d864b3359 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/7503fe94f84243b1950c7a5d864b3359 2024-11-17T08:14:23,674 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 988129a9cf65d53c5913e9c54b149fd7/C of 988129a9cf65d53c5913e9c54b149fd7 into 7503fe94f84243b1950c7a5d864b3359(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:14:23,674 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:23,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T08:14:23,674 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., storeName=988129a9cf65d53c5913e9c54b149fd7/C, priority=13, startTime=1731831263499; duration=0sec 2024-11-17T08:14:23,674 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:23,674 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 988129a9cf65d53c5913e9c54b149fd7:C 2024-11-17T08:14:23,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741893_1069 (size=12301) 2024-11-17T08:14:23,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:23,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:23,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:23,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831323729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:23,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:23,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:23,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831323729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:23,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831323731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:23,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831323731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:23,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:23,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831323733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:23,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:23,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:23,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:23,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831323843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:23,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:23,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:23,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831323843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:23,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:23,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831323843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:23,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:23,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:23,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831323844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831323844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:23,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:23,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:23,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T08:14:24,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:24,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831324054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:24,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:24,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:24,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:24,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:24,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831324054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:24,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:24,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:24,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:24,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:24,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831324054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:24,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:24,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:24,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:24,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:24,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831324054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:24,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:24,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:24,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:24,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:24,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831324054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:24,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:24,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:24,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:24,078 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/bcfea8a65043478f876960630da2132a 2024-11-17T08:14:24,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/0453524bc9194b1c9f0efbad6890ed0a as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/0453524bc9194b1c9f0efbad6890ed0a 2024-11-17T08:14:24,090 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/0453524bc9194b1c9f0efbad6890ed0a, entries=150, sequenceid=304, filesize=12.0 K 2024-11-17T08:14:24,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/2aa124c48f9a4cdb91cf1f3c04cc0c3b as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/2aa124c48f9a4cdb91cf1f3c04cc0c3b 2024-11-17T08:14:24,098 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/2aa124c48f9a4cdb91cf1f3c04cc0c3b, entries=150, sequenceid=304, filesize=12.0 K 2024-11-17T08:14:24,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/bcfea8a65043478f876960630da2132a as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/bcfea8a65043478f876960630da2132a 2024-11-17T08:14:24,108 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/bcfea8a65043478f876960630da2132a, entries=150, sequenceid=304, filesize=12.0 K 2024-11-17T08:14:24,109 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~142.73 KB/146155, heapSize ~404.34 KB/414040, currentSize=40.25 KB/41220 for 988129a9cf65d53c5913e9c54b149fd7 in 596ms, sequenceid=304, compaction requested=false 2024-11-17T08:14:24,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:24,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:24,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-17T08:14:24,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-17T08:14:24,113 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-17T08:14:24,113 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 752 msec 2024-11-17T08:14:24,116 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 759 msec 2024-11-17T08:14:24,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:24,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 988129a9cf65d53c5913e9c54b149fd7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T08:14:24,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=A 2024-11-17T08:14:24,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:24,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=B 2024-11-17T08:14:24,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:24,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=C 2024-11-17T08:14:24,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:24,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/a3fb5e16aa6c4536ab46eb483701bd29 is 50, key is test_row_0/A:col10/1731831263726/Put/seqid=0 2024-11-17T08:14:24,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741894_1070 (size=14741) 2024-11-17T08:14:24,385 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/a3fb5e16aa6c4536ab46eb483701bd29 2024-11-17T08:14:24,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18724, totalHeapSize=42776, min timestamp=1731831264383, max timestamp=1731831264397Num uniques 859; 2024-11-17T08:14:24,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18724, totalHeapSize=42776, min timestamp=1731831264383, max timestamp=1731831264397Num uniques 859; 2024-11-17T08:14:24,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=18724, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-17T08:14:24,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=18724, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-17T08:14:24,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18724, totalHeapSize=42776, min timestamp=1731831264383, max timestamp=1731831264397Num uniques 859; 2024-11-17T08:14:24,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=18724, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-17T08:14:24,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/1561353dd0c74cd6b2e5482ca0f964af is 50, key is test_row_0/B:col10/1731831263726/Put/seqid=0 2024-11-17T08:14:24,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:24,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831324425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:24,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:24,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:24,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:24,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:24,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831324428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:24,441 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:24,441 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:24,441 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:24,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:24,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831324436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:24,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:24,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:24,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:24,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:24,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831324438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:24,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:24,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:24,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:24,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:24,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831324440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:24,446 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:24,446 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:24,446 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:24,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741895_1071 (size=12301) 2024-11-17T08:14:24,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T08:14:24,494 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:14:24,494 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:14:24,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-17T08:14:24,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T08:14:24,496 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:14:24,497 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:14:24,498 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:14:24,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:24,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831324544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:24,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:24,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831324544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:24,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:24,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:24,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:24,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:24,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:24,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:24,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:24,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831324553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:24,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:24,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831324553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:24,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:24,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:24,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:24,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:24,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831324554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:24,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:24,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:24,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:24,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:24,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:24,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T08:14:24,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T08:14:24,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:24,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:24,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:24,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:24,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:24,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:24,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:24,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831324754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:24,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:24,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831324754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:24,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:24,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:24,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:24,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:24,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:24,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:24,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:24,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831324764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:24,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:24,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:24,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831324764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:24,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:24,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:24,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:24,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:24,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:24,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:24,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831324764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:24,769 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:24,769 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:24,769 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:24,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T08:14:24,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:24,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:24,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:24,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:24,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:24,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:24,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T08:14:24,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/1561353dd0c74cd6b2e5482ca0f964af 2024-11-17T08:14:24,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/9911ba0b202040439c40f012edab639f is 50, key is test_row_0/C:col10/1731831263726/Put/seqid=0 2024-11-17T08:14:24,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741896_1072 (size=12301) 2024-11-17T08:14:24,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T08:14:24,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:24,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:24,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:24,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:24,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:24,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:25,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:25,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831325064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:25,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:25,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831325064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:25,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:25,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:25,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:25,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:25,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:25,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:25,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:25,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831325074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:25,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:25,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:25,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:25,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:25,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831325074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:25,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:25,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:25,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:25,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:25,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831325074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:25,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:25,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:25,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:25,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T08:14:25,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:25,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:25,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:25,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:25,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:25,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:25,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T08:14:25,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T08:14:25,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:25,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:25,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:25,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:25,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:25,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:25,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/9911ba0b202040439c40f012edab639f 2024-11-17T08:14:25,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/a3fb5e16aa6c4536ab46eb483701bd29 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/a3fb5e16aa6c4536ab46eb483701bd29 2024-11-17T08:14:25,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/a3fb5e16aa6c4536ab46eb483701bd29, entries=200, sequenceid=317, filesize=14.4 K 2024-11-17T08:14:25,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/1561353dd0c74cd6b2e5482ca0f964af as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/1561353dd0c74cd6b2e5482ca0f964af 2024-11-17T08:14:25,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/1561353dd0c74cd6b2e5482ca0f964af, entries=150, sequenceid=317, filesize=12.0 K 2024-11-17T08:14:25,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/9911ba0b202040439c40f012edab639f as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/9911ba0b202040439c40f012edab639f 2024-11-17T08:14:25,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/9911ba0b202040439c40f012edab639f, entries=150, sequenceid=317, filesize=12.0 K 2024-11-17T08:14:25,306 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 988129a9cf65d53c5913e9c54b149fd7 in 940ms, sequenceid=317, compaction requested=true 2024-11-17T08:14:25,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:25,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 988129a9cf65d53c5913e9c54b149fd7:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:14:25,306 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:14:25,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:25,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 988129a9cf65d53c5913e9c54b149fd7:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:14:25,306 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:14:25,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:25,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 988129a9cf65d53c5913e9c54b149fd7:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:14:25,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:14:25,308 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:14:25,308 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:14:25,308 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): 988129a9cf65d53c5913e9c54b149fd7/A is initiating minor compaction (all files) 2024-11-17T08:14:25,308 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): 988129a9cf65d53c5913e9c54b149fd7/B is initiating minor compaction (all files) 2024-11-17T08:14:25,308 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 988129a9cf65d53c5913e9c54b149fd7/B in TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:25,308 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 988129a9cf65d53c5913e9c54b149fd7/A in TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:25,308 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/2bfb826ce23c49688e6192eac7223278, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/0453524bc9194b1c9f0efbad6890ed0a, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/a3fb5e16aa6c4536ab46eb483701bd29] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp, totalSize=39.0 K 2024-11-17T08:14:25,308 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/89e4c1af56b14cb79c365416a58276d2, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/2aa124c48f9a4cdb91cf1f3c04cc0c3b, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/1561353dd0c74cd6b2e5482ca0f964af] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp, totalSize=36.6 K 2024-11-17T08:14:25,309 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 89e4c1af56b14cb79c365416a58276d2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731831262955 2024-11-17T08:14:25,309 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bfb826ce23c49688e6192eac7223278, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731831262955 2024-11-17T08:14:25,309 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 2aa124c48f9a4cdb91cf1f3c04cc0c3b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731831263063 2024-11-17T08:14:25,309 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0453524bc9194b1c9f0efbad6890ed0a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731831263063 2024-11-17T08:14:25,310 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 1561353dd0c74cd6b2e5482ca0f964af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731831263726 2024-11-17T08:14:25,310 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3fb5e16aa6c4536ab46eb483701bd29, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731831263726 2024-11-17T08:14:25,326 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 988129a9cf65d53c5913e9c54b149fd7#A#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:14:25,326 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 988129a9cf65d53c5913e9c54b149fd7#B#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:14:25,327 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/0907c705159e4240a36b2d2ef5b14f2d is 50, key is test_row_0/B:col10/1731831263726/Put/seqid=0 2024-11-17T08:14:25,327 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/a58d4d1c00d14e76b105d18f01d31ec3 is 50, key is test_row_0/A:col10/1731831263726/Put/seqid=0 2024-11-17T08:14:25,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741897_1073 (size=12983) 2024-11-17T08:14:25,352 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/a58d4d1c00d14e76b105d18f01d31ec3 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/a58d4d1c00d14e76b105d18f01d31ec3 2024-11-17T08:14:25,359 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 988129a9cf65d53c5913e9c54b149fd7/A of 988129a9cf65d53c5913e9c54b149fd7 into a58d4d1c00d14e76b105d18f01d31ec3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:14:25,359 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:25,359 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., storeName=988129a9cf65d53c5913e9c54b149fd7/A, priority=13, startTime=1731831265306; duration=0sec 2024-11-17T08:14:25,359 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:14:25,360 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 988129a9cf65d53c5913e9c54b149fd7:A 2024-11-17T08:14:25,360 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:14:25,361 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:14:25,361 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): 988129a9cf65d53c5913e9c54b149fd7/C is initiating minor compaction (all files) 2024-11-17T08:14:25,361 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 988129a9cf65d53c5913e9c54b149fd7/C in TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:25,361 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/7503fe94f84243b1950c7a5d864b3359, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/bcfea8a65043478f876960630da2132a, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/9911ba0b202040439c40f012edab639f] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp, totalSize=36.6 K 2024-11-17T08:14:25,362 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7503fe94f84243b1950c7a5d864b3359, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731831262955 2024-11-17T08:14:25,362 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcfea8a65043478f876960630da2132a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731831263063 2024-11-17T08:14:25,363 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9911ba0b202040439c40f012edab639f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731831263726 2024-11-17T08:14:25,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741898_1074 (size=12983) 2024-11-17T08:14:25,383 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 988129a9cf65d53c5913e9c54b149fd7#C#compaction#62 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:14:25,383 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/3abd05c7d73b47c0a8f8d925f259977a is 50, key is test_row_0/C:col10/1731831263726/Put/seqid=0 2024-11-17T08:14:25,396 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/0907c705159e4240a36b2d2ef5b14f2d as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/0907c705159e4240a36b2d2ef5b14f2d 2024-11-17T08:14:25,405 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 988129a9cf65d53c5913e9c54b149fd7/B of 988129a9cf65d53c5913e9c54b149fd7 into 0907c705159e4240a36b2d2ef5b14f2d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:14:25,405 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:25,405 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., storeName=988129a9cf65d53c5913e9c54b149fd7/B, priority=13, startTime=1731831265306; duration=0sec 2024-11-17T08:14:25,405 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:25,405 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 988129a9cf65d53c5913e9c54b149fd7:B 2024-11-17T08:14:25,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741899_1075 (size=12983) 2024-11-17T08:14:25,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T08:14:25,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:25,422 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 988129a9cf65d53c5913e9c54b149fd7 3/3 column families, dataSize=140.89 KB heapSize=381.47 KB 2024-11-17T08:14:25,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=A 2024-11-17T08:14:25,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:14:25,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=B 2024-11-17T08:14:25,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:14:25,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=C 2024-11-17T08:14:25,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:14:25,431 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/3abd05c7d73b47c0a8f8d925f259977a as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/3abd05c7d73b47c0a8f8d925f259977a 2024-11-17T08:14:25,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/620cf6a8d9064522a06d80c5c816dfb7 is 50, key is test_row_0/A:col10/1731831264416/Put/seqid=0 2024-11-17T08:14:25,440 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 988129a9cf65d53c5913e9c54b149fd7/C of 988129a9cf65d53c5913e9c54b149fd7 into 3abd05c7d73b47c0a8f8d925f259977a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:14:25,440 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:25,441 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., storeName=988129a9cf65d53c5913e9c54b149fd7/C, priority=13, startTime=1731831265307; duration=0sec 2024-11-17T08:14:25,441 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:25,441 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 988129a9cf65d53c5913e9c54b149fd7:C 2024-11-17T08:14:25,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741900_1076 (size=12301) 2024-11-17T08:14:25,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:25,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:25,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:25,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831325595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:25,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:25,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:25,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:25,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:25,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831325596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:25,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:25,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:25,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:25,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:25,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831325596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:25,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:25,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:25,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:25,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:25,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831325599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:25,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:25,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831325601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:25,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:25,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:25,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:25,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:25,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:25,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:25,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T08:14:25,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:25,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831325704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:25,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:25,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:25,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:25,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:25,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831325704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:25,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:25,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:25,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:25,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:25,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831325714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:25,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:25,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:25,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831325714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:25,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:25,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:25,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:25,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831325715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:25,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:25,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:25,717 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:25,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:25,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:25,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:25,845 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/620cf6a8d9064522a06d80c5c816dfb7 2024-11-17T08:14:25,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/d9e23c1ad5bc43c0a50af50f6714f75d is 50, key is test_row_0/B:col10/1731831264416/Put/seqid=0 2024-11-17T08:14:25,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741901_1077 (size=12301) 2024-11-17T08:14:25,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:25,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831325913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:25,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:25,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:25,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:25,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:25,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831325918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:25,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:25,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:25,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:25,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:25,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:25,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831325924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:25,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831325924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:25,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:25,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:25,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:25,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:25,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:25,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831325924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:25,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:25,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:25,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:25,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:25,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:26,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831326224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:26,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:26,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:26,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831326224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:26,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:26,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:26,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:26,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:26,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:26,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831326233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:26,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:26,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:26,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:26,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:26,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831326244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:26,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:26,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:26,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:26,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:26,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831326244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:26,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:26,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:26,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:26,262 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/d9e23c1ad5bc43c0a50af50f6714f75d 2024-11-17T08:14:26,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/0837e2609b9744d2a82724ce75c994b8 is 50, key is test_row_0/C:col10/1731831264416/Put/seqid=0 2024-11-17T08:14:26,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741902_1078 (size=12301) 2024-11-17T08:14:26,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T08:14:26,705 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/0837e2609b9744d2a82724ce75c994b8 2024-11-17T08:14:26,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/620cf6a8d9064522a06d80c5c816dfb7 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/620cf6a8d9064522a06d80c5c816dfb7 2024-11-17T08:14:26,716 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/620cf6a8d9064522a06d80c5c816dfb7, entries=150, sequenceid=352, filesize=12.0 K 2024-11-17T08:14:26,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/d9e23c1ad5bc43c0a50af50f6714f75d as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/d9e23c1ad5bc43c0a50af50f6714f75d 2024-11-17T08:14:26,723 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/d9e23c1ad5bc43c0a50af50f6714f75d, entries=150, sequenceid=352, filesize=12.0 K 2024-11-17T08:14:26,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/0837e2609b9744d2a82724ce75c994b8 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/0837e2609b9744d2a82724ce75c994b8 2024-11-17T08:14:26,730 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/0837e2609b9744d2a82724ce75c994b8, entries=150, sequenceid=352, filesize=12.0 K 2024-11-17T08:14:26,731 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~381.42 KB/390576, currentSize=46.96 KB/48090 for 988129a9cf65d53c5913e9c54b149fd7 in 1309ms, sequenceid=352, compaction requested=false 2024-11-17T08:14:26,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:26,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:26,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-17T08:14:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-17T08:14:26,735 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-17T08:14:26,735 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2350 sec 2024-11-17T08:14:26,737 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 2.2410 sec 2024-11-17T08:14:26,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:26,748 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 988129a9cf65d53c5913e9c54b149fd7 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-17T08:14:26,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=A 2024-11-17T08:14:26,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:26,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=B 2024-11-17T08:14:26,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:26,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=C 2024-11-17T08:14:26,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:26,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/8730977ca3364008ab85a34d5b038174 is 50, key is test_row_0/A:col10/1731831266748/Put/seqid=0 2024-11-17T08:14:26,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741903_1079 (size=14741) 2024-11-17T08:14:26,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:26,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831326770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:26,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:26,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:26,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:26,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:26,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831326771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:26,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:26,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:26,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:26,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:26,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831326777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:26,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:26,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:26,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:26,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:26,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831326778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:26,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:26,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:26,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:26,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:26,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831326779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:26,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:26,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:26,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:26,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831326884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:26,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:26,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:26,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:26,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831326884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:26,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:26,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:26,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:26,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:26,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831326893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:26,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:26,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:26,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:26,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:26,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:26,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831326894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:26,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831326894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:26,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:26,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:26,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:26,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:26,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:26,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:27,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831327093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:27,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:27,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:27,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:27,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:27,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831327104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:27,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:27,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:27,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831327104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:27,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:27,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:27,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:27,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:27,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:27,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:27,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:27,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831327104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:27,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831327104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:27,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:27,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:27,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:27,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:27,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:27,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:27,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/8730977ca3364008ab85a34d5b038174 2024-11-17T08:14:27,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/03dab20cc16849989deaf3636193fe52 is 50, key is test_row_0/B:col10/1731831266748/Put/seqid=0 2024-11-17T08:14:27,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741904_1080 (size=12301) 2024-11-17T08:14:27,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:27,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831327404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:27,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:27,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:27,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:27,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:27,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831327414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:27,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:27,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:27,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831327414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:27,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:27,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:27,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:27,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:27,416 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:27,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:27,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831327415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:27,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:27,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:27,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831327415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:27,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:27,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:27,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:27,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:27,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:27,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/03dab20cc16849989deaf3636193fe52 2024-11-17T08:14:27,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/fef04c307851497d8cb27bb1df941429 is 50, key is test_row_0/C:col10/1731831266748/Put/seqid=0 2024-11-17T08:14:27,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741905_1081 (size=12301) 2024-11-17T08:14:27,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:27,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831327913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:27,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:27,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:27,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:27,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:27,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831327923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:27,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:27,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831327924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:27,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:27,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:27,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:27,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:27,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:27,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:27,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:27,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831327934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:27,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:27,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:27,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:27,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:27,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831327934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:27,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:27,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:27,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:28,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/fef04c307851497d8cb27bb1df941429 2024-11-17T08:14:28,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/8730977ca3364008ab85a34d5b038174 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/8730977ca3364008ab85a34d5b038174 2024-11-17T08:14:28,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/8730977ca3364008ab85a34d5b038174, entries=200, sequenceid=366, filesize=14.4 K 2024-11-17T08:14:28,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/03dab20cc16849989deaf3636193fe52 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/03dab20cc16849989deaf3636193fe52 2024-11-17T08:14:28,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/03dab20cc16849989deaf3636193fe52, entries=150, sequenceid=366, filesize=12.0 K 2024-11-17T08:14:28,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/fef04c307851497d8cb27bb1df941429 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/fef04c307851497d8cb27bb1df941429 2024-11-17T08:14:28,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/fef04c307851497d8cb27bb1df941429, entries=150, sequenceid=366, filesize=12.0 K 2024-11-17T08:14:28,026 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 988129a9cf65d53c5913e9c54b149fd7 in 1278ms, sequenceid=366, compaction requested=true 2024-11-17T08:14:28,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:28,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 988129a9cf65d53c5913e9c54b149fd7:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:14:28,026 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:14:28,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:28,026 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:14:28,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 988129a9cf65d53c5913e9c54b149fd7:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:14:28,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:28,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 988129a9cf65d53c5913e9c54b149fd7:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:14:28,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:14:28,028 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:14:28,028 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:14:28,028 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): 988129a9cf65d53c5913e9c54b149fd7/B is initiating minor compaction (all files) 2024-11-17T08:14:28,028 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): 988129a9cf65d53c5913e9c54b149fd7/A is initiating minor compaction (all files) 2024-11-17T08:14:28,028 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 988129a9cf65d53c5913e9c54b149fd7/B in TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:28,028 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 988129a9cf65d53c5913e9c54b149fd7/A in TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:28,028 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/a58d4d1c00d14e76b105d18f01d31ec3, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/620cf6a8d9064522a06d80c5c816dfb7, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/8730977ca3364008ab85a34d5b038174] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp, totalSize=39.1 K 2024-11-17T08:14:28,028 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/0907c705159e4240a36b2d2ef5b14f2d, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/d9e23c1ad5bc43c0a50af50f6714f75d, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/03dab20cc16849989deaf3636193fe52] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp, totalSize=36.7 K 2024-11-17T08:14:28,029 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting a58d4d1c00d14e76b105d18f01d31ec3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731831263726 2024-11-17T08:14:28,029 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 0907c705159e4240a36b2d2ef5b14f2d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731831263726 2024-11-17T08:14:28,029 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 620cf6a8d9064522a06d80c5c816dfb7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731831264416 2024-11-17T08:14:28,029 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting d9e23c1ad5bc43c0a50af50f6714f75d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731831264416 2024-11-17T08:14:28,029 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8730977ca3364008ab85a34d5b038174, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731831265592 2024-11-17T08:14:28,029 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 03dab20cc16849989deaf3636193fe52, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731831265592 2024-11-17T08:14:28,039 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 988129a9cf65d53c5913e9c54b149fd7#B#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:14:28,040 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/fced3d05672542888131b06cfe801f46 is 50, key is test_row_0/B:col10/1731831266748/Put/seqid=0 2024-11-17T08:14:28,040 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 988129a9cf65d53c5913e9c54b149fd7#A#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:14:28,041 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/e861662ad76d470a87f648935c5a27c4 is 50, key is test_row_0/A:col10/1731831266748/Put/seqid=0 2024-11-17T08:14:28,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741906_1082 (size=13085) 2024-11-17T08:14:28,061 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/fced3d05672542888131b06cfe801f46 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/fced3d05672542888131b06cfe801f46 2024-11-17T08:14:28,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741907_1083 (size=13085) 2024-11-17T08:14:28,070 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 988129a9cf65d53c5913e9c54b149fd7/B of 988129a9cf65d53c5913e9c54b149fd7 into fced3d05672542888131b06cfe801f46(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:14:28,070 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:28,070 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., storeName=988129a9cf65d53c5913e9c54b149fd7/B, priority=13, startTime=1731831268026; duration=0sec 2024-11-17T08:14:28,070 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:14:28,070 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 988129a9cf65d53c5913e9c54b149fd7:B 2024-11-17T08:14:28,070 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:14:28,073 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/e861662ad76d470a87f648935c5a27c4 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/e861662ad76d470a87f648935c5a27c4 2024-11-17T08:14:28,075 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:14:28,075 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): 988129a9cf65d53c5913e9c54b149fd7/C is initiating minor compaction (all files) 2024-11-17T08:14:28,075 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 988129a9cf65d53c5913e9c54b149fd7/C in TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:28,075 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/3abd05c7d73b47c0a8f8d925f259977a, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/0837e2609b9744d2a82724ce75c994b8, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/fef04c307851497d8cb27bb1df941429] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp, totalSize=36.7 K 2024-11-17T08:14:28,076 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 3abd05c7d73b47c0a8f8d925f259977a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731831263726 2024-11-17T08:14:28,078 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 0837e2609b9744d2a82724ce75c994b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731831264416 2024-11-17T08:14:28,079 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting fef04c307851497d8cb27bb1df941429, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731831265592 2024-11-17T08:14:28,081 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 988129a9cf65d53c5913e9c54b149fd7/A of 988129a9cf65d53c5913e9c54b149fd7 into e861662ad76d470a87f648935c5a27c4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:14:28,081 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:28,081 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., storeName=988129a9cf65d53c5913e9c54b149fd7/A, priority=13, startTime=1731831268026; duration=0sec 2024-11-17T08:14:28,081 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:28,081 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 988129a9cf65d53c5913e9c54b149fd7:A 2024-11-17T08:14:28,089 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 988129a9cf65d53c5913e9c54b149fd7#C#compaction#71 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:14:28,089 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/e8804de7f67b4ee3949fff476d6def36 is 50, key is test_row_0/C:col10/1731831266748/Put/seqid=0 2024-11-17T08:14:28,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741908_1084 (size=13085) 2024-11-17T08:14:28,116 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/e8804de7f67b4ee3949fff476d6def36 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/e8804de7f67b4ee3949fff476d6def36 2024-11-17T08:14:28,124 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 988129a9cf65d53c5913e9c54b149fd7/C of 988129a9cf65d53c5913e9c54b149fd7 into e8804de7f67b4ee3949fff476d6def36(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:14:28,124 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:28,124 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., storeName=988129a9cf65d53c5913e9c54b149fd7/C, priority=13, startTime=1731831268027; duration=0sec 2024-11-17T08:14:28,124 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:28,124 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 988129a9cf65d53c5913e9c54b149fd7:C 2024-11-17T08:14:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T08:14:28,664 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:14:28,665 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:14:28,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-17T08:14:28,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T08:14:28,667 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:14:28,668 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:14:28,668 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:14:28,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T08:14:28,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-17T08:14:28,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:28,820 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 988129a9cf65d53c5913e9c54b149fd7 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-17T08:14:28,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=A 2024-11-17T08:14:28,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:28,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=B 2024-11-17T08:14:28,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:28,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=C 2024-11-17T08:14:28,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:28,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/8821d88ddcd848c3a5f733fc25e39ec5 is 50, key is test_row_0/A:col10/1731831266767/Put/seqid=0 2024-11-17T08:14:28,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741909_1085 (size=12301) 2024-11-17T08:14:28,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:28,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:28,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:28,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831328949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:28,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:28,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:28,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:28,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:28,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831328953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:28,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:28,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:28,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:28,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:28,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831328954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:28,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:28,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831328954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:28,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:28,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:28,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:28,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:28,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831328955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:28,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:28,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:28,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:28,961 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e3d0adb87c01,40631,1731831247805 for 'test_row_1' in 988129a9cf65d53c5913e9c54b149fd7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T08:14:28,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:28,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:28,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:28,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T08:14:29,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:29,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831329064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:29,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:29,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:29,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:29,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:29,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831329064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:29,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:29,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:29,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831329064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:29,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831329064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:29,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:29,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:29,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:29,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:29,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:29,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:29,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:29,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:29,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:29,230 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/8821d88ddcd848c3a5f733fc25e39ec5 2024-11-17T08:14:29,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/2b9f0596f89c4725a5bce85b8f3464e0 is 50, key is test_row_0/B:col10/1731831266767/Put/seqid=0 2024-11-17T08:14:29,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:29,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831329273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:29,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:29,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:29,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:29,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:29,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831329273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:29,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:29,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:29,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:29,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831329273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:29,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:29,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:29,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:29,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831329274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:29,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:29,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:29,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:29,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T08:14:29,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741910_1086 (size=12301) 2024-11-17T08:14:29,304 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/2b9f0596f89c4725a5bce85b8f3464e0 2024-11-17T08:14:29,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/8f9545518c1d4b728fea8cd92bc2bd70 is 50, key is test_row_0/C:col10/1731831266767/Put/seqid=0 2024-11-17T08:14:29,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741911_1087 (size=12301) 2024-11-17T08:14:29,320 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/8f9545518c1d4b728fea8cd92bc2bd70 2024-11-17T08:14:29,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/8821d88ddcd848c3a5f733fc25e39ec5 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/8821d88ddcd848c3a5f733fc25e39ec5 2024-11-17T08:14:29,332 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/8821d88ddcd848c3a5f733fc25e39ec5, entries=150, sequenceid=389, filesize=12.0 K 2024-11-17T08:14:29,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/2b9f0596f89c4725a5bce85b8f3464e0 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/2b9f0596f89c4725a5bce85b8f3464e0 2024-11-17T08:14:29,340 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/2b9f0596f89c4725a5bce85b8f3464e0, entries=150, sequenceid=389, filesize=12.0 K 2024-11-17T08:14:29,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/8f9545518c1d4b728fea8cd92bc2bd70 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/8f9545518c1d4b728fea8cd92bc2bd70 2024-11-17T08:14:29,346 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/8f9545518c1d4b728fea8cd92bc2bd70, entries=150, sequenceid=389, filesize=12.0 K 2024-11-17T08:14:29,347 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 988129a9cf65d53c5913e9c54b149fd7 in 527ms, sequenceid=389, compaction requested=false 2024-11-17T08:14:29,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:29,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:29,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-17T08:14:29,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-17T08:14:29,355 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-17T08:14:29,355 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 682 msec 2024-11-17T08:14:29,358 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 691 msec 2024-11-17T08:14:29,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:29,586 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 988129a9cf65d53c5913e9c54b149fd7 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-17T08:14:29,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=A 2024-11-17T08:14:29,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:29,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=B 2024-11-17T08:14:29,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:29,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=C 2024-11-17T08:14:29,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:29,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/d3cc78fe3e2a40bdaa6e749d1596bf7c is 50, key is test_row_0/A:col10/1731831269584/Put/seqid=0 2024-11-17T08:14:29,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741912_1088 (size=12301) 2024-11-17T08:14:29,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:29,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:29,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831329612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:29,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831329611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:29,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:29,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:29,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:29,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831329618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:29,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:29,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:29,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:29,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:29,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:29,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:29,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:29,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:29,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831329618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:29,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:29,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:29,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:29,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:29,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831329724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:29,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:29,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:29,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:29,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:29,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831329724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:29,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:29,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:29,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:29,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:29,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831329724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:29,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:29,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:29,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:29,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:29,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831329724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:29,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:29,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:29,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:29,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T08:14:29,804 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:14:29,805 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:14:29,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-17T08:14:29,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T08:14:29,808 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:14:29,809 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:14:29,809 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:14:29,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T08:14:29,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:29,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831329934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:29,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:29,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:29,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:29,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:29,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831329935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:29,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:29,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:29,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:29,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:29,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831329935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:29,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:29,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831329935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:29,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:29,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:29,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:29,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:29,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:29,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:29,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-17T08:14:29,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:29,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:29,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:29,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:29,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:29,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:30,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/d3cc78fe3e2a40bdaa6e749d1596bf7c 2024-11-17T08:14:30,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/e164d7959c6648318b6ab4022f893033 is 50, key is test_row_0/B:col10/1731831269584/Put/seqid=0 2024-11-17T08:14:30,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741913_1089 (size=12301) 2024-11-17T08:14:30,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/e164d7959c6648318b6ab4022f893033 2024-11-17T08:14:30,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/9b76b0eae09c476dbb569bd7967f6c6d is 50, key is test_row_0/C:col10/1731831269584/Put/seqid=0 2024-11-17T08:14:30,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741914_1090 (size=12301) 2024-11-17T08:14:30,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-17T08:14:30,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:30,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:30,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:30,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:30,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:30,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:30,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T08:14:30,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:30,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831330243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:30,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:30,244 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:30,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:30,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:30,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831330243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:30,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:30,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:30,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:30,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831330243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:30,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:30,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:30,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:30,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:30,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:30,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831330254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:30,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:30,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:30,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:30,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-17T08:14:30,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:30,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:30,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:30,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:30,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:30,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:30,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-17T08:14:30,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:30,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:30,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:30,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:30,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:30,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:30,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/9b76b0eae09c476dbb569bd7967f6c6d 2024-11-17T08:14:30,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T08:14:30,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/d3cc78fe3e2a40bdaa6e749d1596bf7c as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/d3cc78fe3e2a40bdaa6e749d1596bf7c 2024-11-17T08:14:30,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/d3cc78fe3e2a40bdaa6e749d1596bf7c, entries=150, sequenceid=404, filesize=12.0 K 2024-11-17T08:14:30,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/e164d7959c6648318b6ab4022f893033 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/e164d7959c6648318b6ab4022f893033 2024-11-17T08:14:30,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/e164d7959c6648318b6ab4022f893033, entries=150, sequenceid=404, filesize=12.0 K 2024-11-17T08:14:30,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/9b76b0eae09c476dbb569bd7967f6c6d as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/9b76b0eae09c476dbb569bd7967f6c6d 2024-11-17T08:14:30,465 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/9b76b0eae09c476dbb569bd7967f6c6d, entries=150, sequenceid=404, filesize=12.0 K 2024-11-17T08:14:30,466 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 988129a9cf65d53c5913e9c54b149fd7 in 881ms, sequenceid=404, compaction requested=true 2024-11-17T08:14:30,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:30,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 988129a9cf65d53c5913e9c54b149fd7:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:14:30,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:30,466 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:14:30,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 988129a9cf65d53c5913e9c54b149fd7:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:14:30,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:30,466 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:14:30,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 988129a9cf65d53c5913e9c54b149fd7:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:14:30,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:14:30,467 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:14:30,467 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:14:30,468 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): 988129a9cf65d53c5913e9c54b149fd7/B is initiating minor compaction (all files) 2024-11-17T08:14:30,468 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): 988129a9cf65d53c5913e9c54b149fd7/A is initiating minor compaction (all files) 2024-11-17T08:14:30,468 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 988129a9cf65d53c5913e9c54b149fd7/B in TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:30,468 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 988129a9cf65d53c5913e9c54b149fd7/A in TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:30,468 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/fced3d05672542888131b06cfe801f46, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/2b9f0596f89c4725a5bce85b8f3464e0, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/e164d7959c6648318b6ab4022f893033] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp, totalSize=36.8 K 2024-11-17T08:14:30,468 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/e861662ad76d470a87f648935c5a27c4, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/8821d88ddcd848c3a5f733fc25e39ec5, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/d3cc78fe3e2a40bdaa6e749d1596bf7c] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp, totalSize=36.8 K 2024-11-17T08:14:30,468 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting fced3d05672542888131b06cfe801f46, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731831265592 2024-11-17T08:14:30,469 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting e861662ad76d470a87f648935c5a27c4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731831265592 2024-11-17T08:14:30,469 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b9f0596f89c4725a5bce85b8f3464e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731831266767 2024-11-17T08:14:30,469 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8821d88ddcd848c3a5f733fc25e39ec5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731831266767 2024-11-17T08:14:30,469 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting e164d7959c6648318b6ab4022f893033, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731831268952 2024-11-17T08:14:30,470 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3cc78fe3e2a40bdaa6e749d1596bf7c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731831268952 2024-11-17T08:14:30,484 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 988129a9cf65d53c5913e9c54b149fd7#B#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:14:30,484 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/437d93837147457aacd73ad951515362 is 50, key is test_row_0/B:col10/1731831269584/Put/seqid=0 2024-11-17T08:14:30,490 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 988129a9cf65d53c5913e9c54b149fd7#A#compaction#79 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:14:30,491 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/142298e8120d4658ba51352639d53ab0 is 50, key is test_row_0/A:col10/1731831269584/Put/seqid=0 2024-11-17T08:14:30,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741915_1091 (size=13187) 2024-11-17T08:14:30,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741916_1092 (size=13187) 2024-11-17T08:14:30,514 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/437d93837147457aacd73ad951515362 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/437d93837147457aacd73ad951515362 2024-11-17T08:14:30,526 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/142298e8120d4658ba51352639d53ab0 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/142298e8120d4658ba51352639d53ab0 2024-11-17T08:14:30,529 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 988129a9cf65d53c5913e9c54b149fd7/B of 988129a9cf65d53c5913e9c54b149fd7 into 437d93837147457aacd73ad951515362(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:14:30,529 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:30,529 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., storeName=988129a9cf65d53c5913e9c54b149fd7/B, priority=13, startTime=1731831270466; duration=0sec 2024-11-17T08:14:30,529 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:14:30,530 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 988129a9cf65d53c5913e9c54b149fd7:B 2024-11-17T08:14:30,530 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:14:30,532 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:14:30,532 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): 988129a9cf65d53c5913e9c54b149fd7/C is initiating minor compaction (all files) 2024-11-17T08:14:30,532 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 988129a9cf65d53c5913e9c54b149fd7/C in TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:30,533 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/e8804de7f67b4ee3949fff476d6def36, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/8f9545518c1d4b728fea8cd92bc2bd70, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/9b76b0eae09c476dbb569bd7967f6c6d] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp, totalSize=36.8 K 2024-11-17T08:14:30,533 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting e8804de7f67b4ee3949fff476d6def36, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731831265592 2024-11-17T08:14:30,534 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f9545518c1d4b728fea8cd92bc2bd70, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731831266767 2024-11-17T08:14:30,534 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b76b0eae09c476dbb569bd7967f6c6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731831268952 2024-11-17T08:14:30,536 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 988129a9cf65d53c5913e9c54b149fd7/A of 988129a9cf65d53c5913e9c54b149fd7 into 142298e8120d4658ba51352639d53ab0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:14:30,536 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:30,536 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., storeName=988129a9cf65d53c5913e9c54b149fd7/A, priority=13, startTime=1731831270466; duration=0sec 2024-11-17T08:14:30,536 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:30,537 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 988129a9cf65d53c5913e9c54b149fd7:A 2024-11-17T08:14:30,546 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 988129a9cf65d53c5913e9c54b149fd7#C#compaction#80 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:14:30,546 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/e9fce002f9f6487295045f199b6696f2 is 50, key is test_row_0/C:col10/1731831269584/Put/seqid=0 2024-11-17T08:14:30,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741917_1093 (size=13187) 2024-11-17T08:14:30,563 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/e9fce002f9f6487295045f199b6696f2 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/e9fce002f9f6487295045f199b6696f2 2024-11-17T08:14:30,570 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 988129a9cf65d53c5913e9c54b149fd7/C of 988129a9cf65d53c5913e9c54b149fd7 into e9fce002f9f6487295045f199b6696f2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:14:30,571 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:30,571 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., storeName=988129a9cf65d53c5913e9c54b149fd7/C, priority=13, startTime=1731831270466; duration=0sec 2024-11-17T08:14:30,571 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:30,571 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 988129a9cf65d53c5913e9c54b149fd7:C 2024-11-17T08:14:30,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-17T08:14:30,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:30,579 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 988129a9cf65d53c5913e9c54b149fd7 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-17T08:14:30,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=A 2024-11-17T08:14:30,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:30,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=B 2024-11-17T08:14:30,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:30,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=C 2024-11-17T08:14:30,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:30,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/c4ed7bc50210412ea66064aa1beb5642 is 50, key is test_row_0/A:col10/1731831269616/Put/seqid=0 2024-11-17T08:14:30,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741918_1094 (size=12301) 2024-11-17T08:14:30,602 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/c4ed7bc50210412ea66064aa1beb5642 2024-11-17T08:14:30,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/4e70e42a263b48f88120277b8640f5b9 is 50, key is test_row_0/B:col10/1731831269616/Put/seqid=0 2024-11-17T08:14:30,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741919_1095 (size=12301) 2024-11-17T08:14:30,623 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/4e70e42a263b48f88120277b8640f5b9 2024-11-17T08:14:30,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/61d2546038374eea86d5f384cc74d866 is 50, key is test_row_0/C:col10/1731831269616/Put/seqid=0 2024-11-17T08:14:30,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741920_1096 (size=12301) 2024-11-17T08:14:30,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:30,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:30,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:30,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831330765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:30,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:30,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:30,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:30,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831330766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:30,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:30,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:30,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:30,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:30,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:30,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831330767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:30,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:30,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:30,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:30,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:30,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831330769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:30,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:30,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:30,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:30,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:30,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831330874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:30,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:30,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:30,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831330874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:30,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:30,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:30,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831330874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:30,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:30,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:30,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:30,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:30,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:30,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831330874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:30,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:30,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:30,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:30,881 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:30,881 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:30,881 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:30,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T08:14:30,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:30,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831330973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:30,974 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e3d0adb87c01,40631,1731831247805 for 'test_row_1' in 988129a9cf65d53c5913e9c54b149fd7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T08:14:30,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:30,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:30,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:31,053 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/61d2546038374eea86d5f384cc74d866 2024-11-17T08:14:31,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/c4ed7bc50210412ea66064aa1beb5642 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/c4ed7bc50210412ea66064aa1beb5642 2024-11-17T08:14:31,064 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/c4ed7bc50210412ea66064aa1beb5642, entries=150, sequenceid=425, filesize=12.0 K 2024-11-17T08:14:31,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/4e70e42a263b48f88120277b8640f5b9 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/4e70e42a263b48f88120277b8640f5b9 2024-11-17T08:14:31,070 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/4e70e42a263b48f88120277b8640f5b9, entries=150, sequenceid=425, filesize=12.0 K 2024-11-17T08:14:31,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/61d2546038374eea86d5f384cc74d866 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/61d2546038374eea86d5f384cc74d866 2024-11-17T08:14:31,077 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/61d2546038374eea86d5f384cc74d866, entries=150, sequenceid=425, filesize=12.0 K 2024-11-17T08:14:31,078 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 988129a9cf65d53c5913e9c54b149fd7 in 499ms, sequenceid=425, compaction requested=false 2024-11-17T08:14:31,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:31,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:31,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-17T08:14:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-17T08:14:31,085 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-17T08:14:31,085 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2710 sec 2024-11-17T08:14:31,086 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 988129a9cf65d53c5913e9c54b149fd7 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-17T08:14:31,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:31,091 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.2840 sec 2024-11-17T08:14:31,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=A 2024-11-17T08:14:31,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:31,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=B 2024-11-17T08:14:31,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:31,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=C 2024-11-17T08:14:31,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:31,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/859ec000ff27461da03d0943294c7ce2 is 50, key is test_row_0/A:col10/1731831271084/Put/seqid=0 2024-11-17T08:14:31,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741921_1097 (size=14741) 2024-11-17T08:14:31,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/859ec000ff27461da03d0943294c7ce2 2024-11-17T08:14:31,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/db0c24a7a7574a8593ea56c80b31b3b4 is 50, key is test_row_0/B:col10/1731831271084/Put/seqid=0 2024-11-17T08:14:31,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:31,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831331108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831331107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:31,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:31,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:31,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:31,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:31,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:31,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:31,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831331117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:31,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:31,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:31,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:31,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:31,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831331118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:31,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:31,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:31,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:31,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741922_1098 (size=12301) 2024-11-17T08:14:31,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:31,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831331224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:31,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:31,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:31,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:31,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831331224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:31,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:31,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:31,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:31,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:31,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831331224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:31,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:31,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:31,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:31,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:31,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:31,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831331233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:31,235 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:31,235 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:31,235 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:31,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831331434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:31,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:31,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:31,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:31,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831331434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:31,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:31,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:31,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:31,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831331434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:31,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:31,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:31,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:31,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:31,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831331443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:31,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:31,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:31,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:31,527 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/db0c24a7a7574a8593ea56c80b31b3b4 2024-11-17T08:14:31,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/9f914f2428414958b001b168b5b3d2eb is 50, key is test_row_0/C:col10/1731831271084/Put/seqid=0 2024-11-17T08:14:31,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741923_1099 (size=12301) 2024-11-17T08:14:31,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/9f914f2428414958b001b168b5b3d2eb 2024-11-17T08:14:31,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/859ec000ff27461da03d0943294c7ce2 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/859ec000ff27461da03d0943294c7ce2 2024-11-17T08:14:31,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/859ec000ff27461da03d0943294c7ce2, entries=200, sequenceid=443, filesize=14.4 K 2024-11-17T08:14:31,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/db0c24a7a7574a8593ea56c80b31b3b4 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/db0c24a7a7574a8593ea56c80b31b3b4 2024-11-17T08:14:31,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/db0c24a7a7574a8593ea56c80b31b3b4, entries=150, sequenceid=443, filesize=12.0 K 2024-11-17T08:14:31,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/9f914f2428414958b001b168b5b3d2eb as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/9f914f2428414958b001b168b5b3d2eb 2024-11-17T08:14:31,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/9f914f2428414958b001b168b5b3d2eb, entries=150, sequenceid=443, filesize=12.0 K 2024-11-17T08:14:31,573 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 988129a9cf65d53c5913e9c54b149fd7 in 487ms, sequenceid=443, compaction requested=true 2024-11-17T08:14:31,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:31,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 988129a9cf65d53c5913e9c54b149fd7:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:14:31,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:31,573 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:14:31,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 988129a9cf65d53c5913e9c54b149fd7:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:14:31,573 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:14:31,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:31,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 988129a9cf65d53c5913e9c54b149fd7:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:14:31,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:14:31,574 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:14:31,574 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:14:31,574 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): 988129a9cf65d53c5913e9c54b149fd7/B is initiating minor compaction (all files) 2024-11-17T08:14:31,574 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): 988129a9cf65d53c5913e9c54b149fd7/A is initiating minor compaction (all files) 2024-11-17T08:14:31,574 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 988129a9cf65d53c5913e9c54b149fd7/B in TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:31,574 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 988129a9cf65d53c5913e9c54b149fd7/A in TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:31,574 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/437d93837147457aacd73ad951515362, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/4e70e42a263b48f88120277b8640f5b9, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/db0c24a7a7574a8593ea56c80b31b3b4] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp, totalSize=36.9 K 2024-11-17T08:14:31,574 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/142298e8120d4658ba51352639d53ab0, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/c4ed7bc50210412ea66064aa1beb5642, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/859ec000ff27461da03d0943294c7ce2] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp, totalSize=39.3 K 2024-11-17T08:14:31,575 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 437d93837147457aacd73ad951515362, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731831268952 2024-11-17T08:14:31,575 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 142298e8120d4658ba51352639d53ab0, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731831268952 2024-11-17T08:14:31,575 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e70e42a263b48f88120277b8640f5b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1731831269609 2024-11-17T08:14:31,575 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4ed7bc50210412ea66064aa1beb5642, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1731831269609 2024-11-17T08:14:31,575 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting db0c24a7a7574a8593ea56c80b31b3b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731831271084 2024-11-17T08:14:31,575 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 859ec000ff27461da03d0943294c7ce2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731831270766 2024-11-17T08:14:31,584 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 988129a9cf65d53c5913e9c54b149fd7#A#compaction#87 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:14:31,584 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/69b7562a4c6c4575b7a078d5a654056c is 50, key is test_row_0/A:col10/1731831271084/Put/seqid=0 2024-11-17T08:14:31,585 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 988129a9cf65d53c5913e9c54b149fd7#B#compaction#88 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:14:31,586 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/969ce8656a624fbeb757784da0bc62a8 is 50, key is test_row_0/B:col10/1731831271084/Put/seqid=0 2024-11-17T08:14:31,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741925_1101 (size=13289) 2024-11-17T08:14:31,602 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/969ce8656a624fbeb757784da0bc62a8 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/969ce8656a624fbeb757784da0bc62a8 2024-11-17T08:14:31,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741924_1100 (size=13289) 2024-11-17T08:14:31,609 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 988129a9cf65d53c5913e9c54b149fd7/B of 988129a9cf65d53c5913e9c54b149fd7 into 969ce8656a624fbeb757784da0bc62a8(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:14:31,609 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:31,609 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., storeName=988129a9cf65d53c5913e9c54b149fd7/B, priority=13, startTime=1731831271573; duration=0sec 2024-11-17T08:14:31,610 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:14:31,610 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 988129a9cf65d53c5913e9c54b149fd7:B 2024-11-17T08:14:31,610 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:14:31,613 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/69b7562a4c6c4575b7a078d5a654056c as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/69b7562a4c6c4575b7a078d5a654056c 2024-11-17T08:14:31,615 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:14:31,615 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): 988129a9cf65d53c5913e9c54b149fd7/C is initiating minor compaction (all files) 2024-11-17T08:14:31,616 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 988129a9cf65d53c5913e9c54b149fd7/C in TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:31,616 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/e9fce002f9f6487295045f199b6696f2, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/61d2546038374eea86d5f384cc74d866, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/9f914f2428414958b001b168b5b3d2eb] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp, totalSize=36.9 K 2024-11-17T08:14:31,616 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting e9fce002f9f6487295045f199b6696f2, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731831268952 2024-11-17T08:14:31,617 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 61d2546038374eea86d5f384cc74d866, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1731831269609 2024-11-17T08:14:31,618 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f914f2428414958b001b168b5b3d2eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731831271084 2024-11-17T08:14:31,621 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 988129a9cf65d53c5913e9c54b149fd7/A of 988129a9cf65d53c5913e9c54b149fd7 into 69b7562a4c6c4575b7a078d5a654056c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:14:31,621 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:31,621 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., storeName=988129a9cf65d53c5913e9c54b149fd7/A, priority=13, startTime=1731831271573; duration=0sec 2024-11-17T08:14:31,621 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:31,621 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 988129a9cf65d53c5913e9c54b149fd7:A 2024-11-17T08:14:31,638 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 988129a9cf65d53c5913e9c54b149fd7#C#compaction#89 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:14:31,639 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/7ec8e6a2957a42a2931dab4a5e247ade is 50, key is test_row_0/C:col10/1731831271084/Put/seqid=0 2024-11-17T08:14:31,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741926_1102 (size=13289) 2024-11-17T08:14:31,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:31,747 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 988129a9cf65d53c5913e9c54b149fd7 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-17T08:14:31,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=A 2024-11-17T08:14:31,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:31,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=B 2024-11-17T08:14:31,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:31,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=C 2024-11-17T08:14:31,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:31,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/1be9d4d9bf5341deb6f816c29fbd1b75 is 50, key is test_row_0/A:col10/1731831271111/Put/seqid=0 2024-11-17T08:14:31,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741927_1103 (size=14741) 2024-11-17T08:14:31,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:31,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831331762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:31,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:31,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:31,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:31,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:31,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831331765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:31,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:31,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:31,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:31,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:31,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831331765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:31,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:31,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:31,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:31,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831331766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:31,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:31,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:31,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:31,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:31,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:31,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831331874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:31,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:31,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:31,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:31,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:31,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831331874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:31,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:31,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:31,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:31,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:31,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831331874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:31,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:31,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:31,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:31,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:31,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831331883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:31,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:31,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:31,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:31,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T08:14:31,953 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:14:31,954 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:14:31,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-17T08:14:31,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T08:14:31,956 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:14:31,956 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:14:31,956 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:14:32,049 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/7ec8e6a2957a42a2931dab4a5e247ade as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/7ec8e6a2957a42a2931dab4a5e247ade 2024-11-17T08:14:32,054 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 988129a9cf65d53c5913e9c54b149fd7/C of 988129a9cf65d53c5913e9c54b149fd7 into 7ec8e6a2957a42a2931dab4a5e247ade(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:14:32,054 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:32,054 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., storeName=988129a9cf65d53c5913e9c54b149fd7/C, priority=13, startTime=1731831271573; duration=0sec 2024-11-17T08:14:32,054 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:32,054 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 988129a9cf65d53c5913e9c54b149fd7:C 2024-11-17T08:14:32,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T08:14:32,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:32,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831332083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:32,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:32,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:32,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831332083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:32,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:32,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:32,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:32,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:32,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:32,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:32,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831332083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:32,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:32,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:32,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:32,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:32,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831332094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:32,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:32,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:32,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:32,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-17T08:14:32,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:32,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:32,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:32,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:32,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:32,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:32,156 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/1be9d4d9bf5341deb6f816c29fbd1b75 2024-11-17T08:14:32,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/d407444192f64469a3c690ed751b2f8f is 50, key is test_row_0/B:col10/1731831271111/Put/seqid=0 2024-11-17T08:14:32,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741928_1104 (size=12301) 2024-11-17T08:14:32,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-17T08:14:32,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:32,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:32,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:32,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:32,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:32,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:32,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T08:14:32,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:32,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:32,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831332393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:32,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831332393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:32,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:32,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:32,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831332393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:32,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:32,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:32,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:32,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:32,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:32,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:32,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:32,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:32,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:32,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831332404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:32,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:32,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:32,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:32,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-17T08:14:32,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:32,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:32,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:32,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:32,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:32,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:32,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-17T08:14:32,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:32,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:32,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:32,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:32,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:32,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:32,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/d407444192f64469a3c690ed751b2f8f 2024-11-17T08:14:32,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/815448df7e8c46c7816105248ab1cce1 is 50, key is test_row_0/C:col10/1731831271111/Put/seqid=0 2024-11-17T08:14:32,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T08:14:32,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741929_1105 (size=12301) 2024-11-17T08:14:32,590 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/815448df7e8c46c7816105248ab1cce1 2024-11-17T08:14:32,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/1be9d4d9bf5341deb6f816c29fbd1b75 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/1be9d4d9bf5341deb6f816c29fbd1b75 2024-11-17T08:14:32,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/1be9d4d9bf5341deb6f816c29fbd1b75, entries=200, sequenceid=462, filesize=14.4 K 2024-11-17T08:14:32,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/d407444192f64469a3c690ed751b2f8f as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/d407444192f64469a3c690ed751b2f8f 2024-11-17T08:14:32,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/d407444192f64469a3c690ed751b2f8f, entries=150, sequenceid=462, filesize=12.0 K 2024-11-17T08:14:32,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/815448df7e8c46c7816105248ab1cce1 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/815448df7e8c46c7816105248ab1cce1 2024-11-17T08:14:32,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/815448df7e8c46c7816105248ab1cce1, entries=150, sequenceid=462, filesize=12.0 K 2024-11-17T08:14:32,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 988129a9cf65d53c5913e9c54b149fd7 in 876ms, sequenceid=462, compaction requested=false 2024-11-17T08:14:32,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:32,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-17T08:14:32,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:32,722 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 988129a9cf65d53c5913e9c54b149fd7 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-17T08:14:32,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=A 2024-11-17T08:14:32,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:32,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=B 2024-11-17T08:14:32,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:32,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=C 2024-11-17T08:14:32,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:32,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/ada9205f3b6c4bfdb1f6e5408eb81cfb is 50, key is test_row_0/A:col10/1731831271760/Put/seqid=0 2024-11-17T08:14:32,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741930_1106 (size=12301) 2024-11-17T08:14:32,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:32,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:32,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:32,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831332936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:32,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:32,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:32,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831332936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:32,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:32,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:32,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:32,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:32,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:32,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:32,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:32,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831332937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:32,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831332937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:32,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:32,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:32,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:32,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:32,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:32,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:33,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:33,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831333044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:33,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:33,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:33,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:33,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831333044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:33,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:33,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:33,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:33,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:33,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831333044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:33,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:33,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:33,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:33,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:33,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:33,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831333044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:33,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:33,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:33,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:33,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T08:14:33,132 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/ada9205f3b6c4bfdb1f6e5408eb81cfb 2024-11-17T08:14:33,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/42e5b9b59c724837b3fe0f5d9de4a735 is 50, key is test_row_0/B:col10/1731831271760/Put/seqid=0 2024-11-17T08:14:33,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741931_1107 (size=12301) 2024-11-17T08:14:33,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:33,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831333254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:33,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:33,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831333254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:33,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:33,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:33,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:33,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:33,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:33,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:33,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831333254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:33,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:33,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:33,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:33,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:33,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:33,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831333254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:33,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:33,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:33,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:33,546 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/42e5b9b59c724837b3fe0f5d9de4a735 2024-11-17T08:14:33,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/5aa9f30265ac4b6086e0262450e5f58e is 50, key is test_row_0/C:col10/1731831271760/Put/seqid=0 2024-11-17T08:14:33,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741932_1108 (size=12301) 2024-11-17T08:14:33,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:33,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831333564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:33,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:33,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:33,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831333564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:33,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:33,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:33,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:33,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:33,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:33,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:33,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831333564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:33,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:33,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:33,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:33,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:33,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831333564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:33,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:33,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:33,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:33,959 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/5aa9f30265ac4b6086e0262450e5f58e 2024-11-17T08:14:33,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/ada9205f3b6c4bfdb1f6e5408eb81cfb as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/ada9205f3b6c4bfdb1f6e5408eb81cfb 2024-11-17T08:14:33,969 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/ada9205f3b6c4bfdb1f6e5408eb81cfb, entries=150, sequenceid=478, filesize=12.0 K 2024-11-17T08:14:33,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/42e5b9b59c724837b3fe0f5d9de4a735 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/42e5b9b59c724837b3fe0f5d9de4a735 2024-11-17T08:14:33,975 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/42e5b9b59c724837b3fe0f5d9de4a735, entries=150, sequenceid=478, filesize=12.0 K 2024-11-17T08:14:33,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/5aa9f30265ac4b6086e0262450e5f58e as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/5aa9f30265ac4b6086e0262450e5f58e 2024-11-17T08:14:33,981 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/5aa9f30265ac4b6086e0262450e5f58e, entries=150, sequenceid=478, filesize=12.0 K 2024-11-17T08:14:33,982 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 988129a9cf65d53c5913e9c54b149fd7 in 1260ms, sequenceid=478, compaction requested=true 2024-11-17T08:14:33,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:33,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:33,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-17T08:14:33,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-17T08:14:33,985 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-17T08:14:33,985 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0270 sec 2024-11-17T08:14:33,987 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 2.0320 sec 2024-11-17T08:14:34,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:34,076 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 988129a9cf65d53c5913e9c54b149fd7 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-17T08:14:34,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=A 2024-11-17T08:14:34,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:34,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=B 2024-11-17T08:14:34,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:34,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=C 2024-11-17T08:14:34,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:34,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/89d7419bbdd648bbb8ed5e71136b783a is 50, key is test_row_0/A:col10/1731831272921/Put/seqid=0 2024-11-17T08:14:34,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741933_1109 (size=17181) 2024-11-17T08:14:34,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:34,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831334087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:34,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:34,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:34,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831334088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:34,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:34,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:34,094 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:34,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:34,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:34,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:34,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831334090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:34,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:34,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:34,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:34,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:34,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831334094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:34,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:34,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:34,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:34,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T08:14:34,103 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:14:34,104 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:14:34,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-17T08:14:34,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T08:14:34,106 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:14:34,106 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:14:34,106 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:14:34,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:34,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831334204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:34,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:34,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:34,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831334204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:34,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:34,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:34,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:34,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:34,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:34,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831334204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:34,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:34,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:34,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:34,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:34,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:34,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831334204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:34,208 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:34,208 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:34,208 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:34,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T08:14:34,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T08:14:34,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:34,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:34,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:34,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:34,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:34,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:34,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:34,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:34,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831334413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:34,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831334413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:34,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:34,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:34,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:34,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:34,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831334413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:34,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:34,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:34,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831334413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:34,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:34,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:34,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:34,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:34,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:34,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:34,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:34,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:34,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T08:14:34,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/89d7419bbdd648bbb8ed5e71136b783a 2024-11-17T08:14:34,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T08:14:34,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:34,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:34,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:34,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:34,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:34,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:34,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/e43c8fcaf6744b95a76d810c117ad6d9 is 50, key is test_row_0/B:col10/1731831272921/Put/seqid=0 2024-11-17T08:14:34,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741934_1110 (size=12301) 2024-11-17T08:14:34,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T08:14:34,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:34,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:34,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:34,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:34,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:34,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:34,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:34,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831334724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:34,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:34,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:34,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831334724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:34,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:34,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:34,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:34,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:34,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:34,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:34,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831334724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:34,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:34,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:34,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:34,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:34,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831334724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:34,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:34,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:34,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:34,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T08:14:34,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T08:14:34,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:34,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:34,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:34,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:34,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:34,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/e43c8fcaf6744b95a76d810c117ad6d9 2024-11-17T08:14:34,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/a8169c3c34434756bba3457700e70c80 is 50, key is test_row_0/C:col10/1731831272921/Put/seqid=0 2024-11-17T08:14:34,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741935_1111 (size=12301) 2024-11-17T08:14:34,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T08:14:34,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:34,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:34,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:34,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:34,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:34,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:34,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:34,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40706 deadline: 1731831334994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:34,995 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e3d0adb87c01,40631,1731831247805 for 'test_row_1' in 988129a9cf65d53c5913e9c54b149fd7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T08:14:34,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:34,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:34,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:35,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T08:14:35,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:35,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:35,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:35,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:35,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:35,103 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-17T08:14:35,104 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:14:35,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:35,104 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-17T08:14:35,104 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:14:35,106 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-17T08:14:35,106 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:14:35,107 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-17T08:14:35,107 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:14:35,108 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:14:35,108 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:14:35,108 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:14:35,108 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:14:35,109 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:14:35,109 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:14:35,109 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:14:35,109 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:14:35,109 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:14:35,109 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:14:35,109 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:14:35,109 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:14:35,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:35,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:35,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40730 deadline: 1731831335235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:35,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40724 deadline: 1731831335236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:35,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:35,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:35,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40764 deadline: 1731831335237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:35,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:35,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:35,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:35,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40708 deadline: 1731831335237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:35,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:35,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:35,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:35,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:35,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:35,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:35,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:35,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:35,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:14:35,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T08:14:35,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T08:14:35,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:35,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:35,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:35,257 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:35,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:35,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:35,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/a8169c3c34434756bba3457700e70c80 2024-11-17T08:14:35,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/89d7419bbdd648bbb8ed5e71136b783a as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/89d7419bbdd648bbb8ed5e71136b783a 2024-11-17T08:14:35,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/89d7419bbdd648bbb8ed5e71136b783a, entries=250, sequenceid=497, filesize=16.8 K 2024-11-17T08:14:35,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/e43c8fcaf6744b95a76d810c117ad6d9 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/e43c8fcaf6744b95a76d810c117ad6d9 2024-11-17T08:14:35,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/e43c8fcaf6744b95a76d810c117ad6d9, entries=150, sequenceid=497, filesize=12.0 K 2024-11-17T08:14:35,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/a8169c3c34434756bba3457700e70c80 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/a8169c3c34434756bba3457700e70c80 2024-11-17T08:14:35,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/a8169c3c34434756bba3457700e70c80, entries=150, sequenceid=497, filesize=12.0 K 2024-11-17T08:14:35,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 988129a9cf65d53c5913e9c54b149fd7 in 1289ms, sequenceid=497, compaction requested=true 2024-11-17T08:14:35,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:35,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 988129a9cf65d53c5913e9c54b149fd7:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:14:35,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:35,365 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T08:14:35,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 988129a9cf65d53c5913e9c54b149fd7:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:14:35,365 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T08:14:35,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:35,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 988129a9cf65d53c5913e9c54b149fd7:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:14:35,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:14:35,367 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T08:14:35,367 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57512 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T08:14:35,367 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): 988129a9cf65d53c5913e9c54b149fd7/B is initiating minor compaction (all files) 2024-11-17T08:14:35,367 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): 988129a9cf65d53c5913e9c54b149fd7/A is initiating minor compaction (all files) 2024-11-17T08:14:35,367 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 988129a9cf65d53c5913e9c54b149fd7/B in TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:35,367 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 988129a9cf65d53c5913e9c54b149fd7/A in TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:35,367 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/969ce8656a624fbeb757784da0bc62a8, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/d407444192f64469a3c690ed751b2f8f, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/42e5b9b59c724837b3fe0f5d9de4a735, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/e43c8fcaf6744b95a76d810c117ad6d9] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp, totalSize=49.0 K 2024-11-17T08:14:35,367 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/69b7562a4c6c4575b7a078d5a654056c, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/1be9d4d9bf5341deb6f816c29fbd1b75, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/ada9205f3b6c4bfdb1f6e5408eb81cfb, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/89d7419bbdd648bbb8ed5e71136b783a] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp, totalSize=56.2 K 2024-11-17T08:14:35,368 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 969ce8656a624fbeb757784da0bc62a8, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731831271084 2024-11-17T08:14:35,368 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 69b7562a4c6c4575b7a078d5a654056c, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731831271084 2024-11-17T08:14:35,368 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting d407444192f64469a3c690ed751b2f8f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1731831271111 2024-11-17T08:14:35,368 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 1be9d4d9bf5341deb6f816c29fbd1b75, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1731831271111 2024-11-17T08:14:35,368 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42e5b9b59c724837b3fe0f5d9de4a735, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1731831271760 2024-11-17T08:14:35,368 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting ada9205f3b6c4bfdb1f6e5408eb81cfb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1731831271760 2024-11-17T08:14:35,369 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting e43c8fcaf6744b95a76d810c117ad6d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1731831272921 2024-11-17T08:14:35,369 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 89d7419bbdd648bbb8ed5e71136b783a, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1731831272921 2024-11-17T08:14:35,385 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 988129a9cf65d53c5913e9c54b149fd7#B#compaction#99 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:14:35,386 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/fc78dd906ceb492c9abca10f8d68b02b is 50, key is test_row_0/B:col10/1731831272921/Put/seqid=0 2024-11-17T08:14:35,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741936_1112 (size=13425) 2024-11-17T08:14:35,397 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/fc78dd906ceb492c9abca10f8d68b02b as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/fc78dd906ceb492c9abca10f8d68b02b 2024-11-17T08:14:35,404 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 988129a9cf65d53c5913e9c54b149fd7#A#compaction#100 average throughput is 0.26 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:14:35,404 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 988129a9cf65d53c5913e9c54b149fd7/B of 988129a9cf65d53c5913e9c54b149fd7 into fc78dd906ceb492c9abca10f8d68b02b(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:14:35,404 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:35,404 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., storeName=988129a9cf65d53c5913e9c54b149fd7/B, priority=12, startTime=1731831275365; duration=0sec 2024-11-17T08:14:35,404 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:14:35,404 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 988129a9cf65d53c5913e9c54b149fd7:B 2024-11-17T08:14:35,404 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/f3a061e8baad4ace871376eda9ce43b3 is 50, key is test_row_0/A:col10/1731831272921/Put/seqid=0 2024-11-17T08:14:35,404 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T08:14:35,406 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T08:14:35,406 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): 988129a9cf65d53c5913e9c54b149fd7/C is initiating minor compaction (all files) 2024-11-17T08:14:35,406 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 988129a9cf65d53c5913e9c54b149fd7/C in TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:35,406 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/7ec8e6a2957a42a2931dab4a5e247ade, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/815448df7e8c46c7816105248ab1cce1, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/5aa9f30265ac4b6086e0262450e5f58e, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/a8169c3c34434756bba3457700e70c80] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp, totalSize=49.0 K 2024-11-17T08:14:35,407 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ec8e6a2957a42a2931dab4a5e247ade, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731831271084 2024-11-17T08:14:35,407 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 815448df7e8c46c7816105248ab1cce1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1731831271111 2024-11-17T08:14:35,407 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5aa9f30265ac4b6086e0262450e5f58e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1731831271760 2024-11-17T08:14:35,408 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8169c3c34434756bba3457700e70c80, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1731831272921 2024-11-17T08:14:35,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741937_1113 (size=13425) 2024-11-17T08:14:35,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-17T08:14:35,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:35,412 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing 988129a9cf65d53c5913e9c54b149fd7 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-17T08:14:35,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=A 2024-11-17T08:14:35,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:35,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=B 2024-11-17T08:14:35,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:35,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=C 2024-11-17T08:14:35,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:35,413 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/f3a061e8baad4ace871376eda9ce43b3 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/f3a061e8baad4ace871376eda9ce43b3 2024-11-17T08:14:35,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/ada1d0e952c844b9b35611376077bd04 is 50, key is test_row_0/A:col10/1731831274087/Put/seqid=0 2024-11-17T08:14:35,420 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 988129a9cf65d53c5913e9c54b149fd7/A of 988129a9cf65d53c5913e9c54b149fd7 into f3a061e8baad4ace871376eda9ce43b3(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:14:35,420 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:35,420 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., storeName=988129a9cf65d53c5913e9c54b149fd7/A, priority=12, startTime=1731831275365; duration=0sec 2024-11-17T08:14:35,420 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:35,420 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 988129a9cf65d53c5913e9c54b149fd7:A 2024-11-17T08:14:35,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741938_1114 (size=12301) 2024-11-17T08:14:35,424 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/ada1d0e952c844b9b35611376077bd04 2024-11-17T08:14:35,430 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 988129a9cf65d53c5913e9c54b149fd7#C#compaction#102 average throughput is 0.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:14:35,431 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/0351b21016194a80aec032ca13a2aa3a is 50, key is test_row_0/C:col10/1731831272921/Put/seqid=0 2024-11-17T08:14:35,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/29a536f1e6d44b83962906ec5be86020 is 50, key is test_row_0/B:col10/1731831274087/Put/seqid=0 2024-11-17T08:14:35,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741939_1115 (size=13425) 2024-11-17T08:14:35,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741940_1116 (size=12301) 2024-11-17T08:14:35,442 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/0351b21016194a80aec032ca13a2aa3a as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/0351b21016194a80aec032ca13a2aa3a 2024-11-17T08:14:35,448 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 988129a9cf65d53c5913e9c54b149fd7/C of 988129a9cf65d53c5913e9c54b149fd7 into 0351b21016194a80aec032ca13a2aa3a(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:14:35,448 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:35,448 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., storeName=988129a9cf65d53c5913e9c54b149fd7/C, priority=12, startTime=1731831275365; duration=0sec 2024-11-17T08:14:35,448 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:35,448 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 988129a9cf65d53c5913e9c54b149fd7:C 2024-11-17T08:14:35,838 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/29a536f1e6d44b83962906ec5be86020 2024-11-17T08:14:35,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/db5e377a92a4497f86d595365acbbc47 is 50, key is test_row_0/C:col10/1731831274087/Put/seqid=0 2024-11-17T08:14:35,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741941_1117 (size=12301) 2024-11-17T08:14:36,110 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T08:14:36,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:36,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. as already flushing 2024-11-17T08:14:36,252 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-17T08:14:36,252 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:14:36,252 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:14:36,253 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:14:36,253 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:14:36,256 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-17T08:14:36,257 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:14:36,257 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-17T08:14:36,257 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:14:36,257 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:14:36,257 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:14:36,257 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:14:36,257 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:14:36,258 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:14:36,258 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:14:36,259 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-17T08:14:36,259 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:14:36,259 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:14:36,259 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/db5e377a92a4497f86d595365acbbc47 2024-11-17T08:14:36,259 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:14:36,259 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:14:36,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T08:14:36,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/ada1d0e952c844b9b35611376077bd04 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/ada1d0e952c844b9b35611376077bd04 2024-11-17T08:14:36,277 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/ada1d0e952c844b9b35611376077bd04, entries=150, sequenceid=511, filesize=12.0 K 2024-11-17T08:14:36,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/29a536f1e6d44b83962906ec5be86020 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/29a536f1e6d44b83962906ec5be86020 2024-11-17T08:14:36,286 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/29a536f1e6d44b83962906ec5be86020, entries=150, sequenceid=511, filesize=12.0 K 2024-11-17T08:14:36,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/db5e377a92a4497f86d595365acbbc47 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/db5e377a92a4497f86d595365acbbc47 2024-11-17T08:14:36,293 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/db5e377a92a4497f86d595365acbbc47, entries=150, sequenceid=511, filesize=12.0 K 2024-11-17T08:14:36,295 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=26.84 KB/27480 for 988129a9cf65d53c5913e9c54b149fd7 in 883ms, sequenceid=511, compaction requested=false 2024-11-17T08:14:36,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for 988129a9cf65d53c5913e9c54b149fd7: 2024-11-17T08:14:36,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:36,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-17T08:14:36,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-17T08:14:36,299 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-17T08:14:36,299 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1910 sec 2024-11-17T08:14:36,303 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 2.1960 sec 2024-11-17T08:14:38,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-17T08:14:38,284 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:14:45,117 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-17T08:14:45,117 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:14:45,117 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:14:45,117 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:14:45,117 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:14:45,118 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T08:14:45,118 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 86 2024-11-17T08:14:45,118 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-17T08:14:45,118 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-17T08:14:45,118 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-11-17T08:14:45,118 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-17T08:14:45,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T08:14:45,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3002 2024-11-17T08:14:45,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2887 2024-11-17T08:14:45,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T08:14:45,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1262 2024-11-17T08:14:45,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3778 rows 2024-11-17T08:14:45,119 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1265 2024-11-17T08:14:45,120 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3790 rows 2024-11-17T08:14:45,120 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T08:14:45,120 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T08:14:45,120 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:14:45,120 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:14:45,120 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:14:45,125 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-17T08:14:45,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T08:14:45,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-17T08:14:45,131 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731831285131"}]},"ts":"1731831285131"} 2024-11-17T08:14:45,133 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T08:14:45,133 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T08:14:45,134 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T08:14:45,139 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=988129a9cf65d53c5913e9c54b149fd7, UNASSIGN}] 2024-11-17T08:14:45,140 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=988129a9cf65d53c5913e9c54b149fd7, UNASSIGN 2024-11-17T08:14:45,141 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=988129a9cf65d53c5913e9c54b149fd7, regionState=CLOSING, regionLocation=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:45,143 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=988129a9cf65d53c5913e9c54b149fd7, UNASSIGN because future has completed 2024-11-17T08:14:45,144 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T08:14:45,144 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805}] 2024-11-17T08:14:45,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-17T08:14:45,300 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close 988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:45,300 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T08:14:45,301 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing 988129a9cf65d53c5913e9c54b149fd7, disabling compactions & flushes 2024-11-17T08:14:45,301 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:45,301 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:45,301 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. after waiting 0 ms 2024-11-17T08:14:45,301 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:45,302 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(2902): Flushing 988129a9cf65d53c5913e9c54b149fd7 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-17T08:14:45,302 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=A 2024-11-17T08:14:45,302 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:45,302 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=B 2024-11-17T08:14:45,302 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:45,302 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 988129a9cf65d53c5913e9c54b149fd7, store=C 2024-11-17T08:14:45,302 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:45,308 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/73df514594ae4742ab560ba3b9f4c4c6 is 50, key is test_row_0/A:col10/1731831276254/Put/seqid=0 2024-11-17T08:14:45,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741942_1118 (size=12301) 2024-11-17T08:14:45,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-17T08:14:45,714 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/73df514594ae4742ab560ba3b9f4c4c6 2024-11-17T08:14:45,724 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/97d392b309bf43e08e065c6e4efd04e3 is 50, key is test_row_0/B:col10/1731831276254/Put/seqid=0 2024-11-17T08:14:45,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741943_1119 (size=12301) 2024-11-17T08:14:45,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-17T08:14:46,129 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/97d392b309bf43e08e065c6e4efd04e3 2024-11-17T08:14:46,141 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/b4ac01c6b1c545fc9e0c65afad4538fc is 50, key is test_row_0/C:col10/1731831276254/Put/seqid=0 2024-11-17T08:14:46,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741944_1120 (size=12301) 2024-11-17T08:14:46,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-17T08:14:46,548 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/b4ac01c6b1c545fc9e0c65afad4538fc 2024-11-17T08:14:46,559 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/A/73df514594ae4742ab560ba3b9f4c4c6 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/73df514594ae4742ab560ba3b9f4c4c6 2024-11-17T08:14:46,565 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/73df514594ae4742ab560ba3b9f4c4c6, entries=150, sequenceid=521, filesize=12.0 K 2024-11-17T08:14:46,567 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/B/97d392b309bf43e08e065c6e4efd04e3 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/97d392b309bf43e08e065c6e4efd04e3 2024-11-17T08:14:46,572 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/97d392b309bf43e08e065c6e4efd04e3, entries=150, sequenceid=521, filesize=12.0 K 2024-11-17T08:14:46,573 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/.tmp/C/b4ac01c6b1c545fc9e0c65afad4538fc as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/b4ac01c6b1c545fc9e0c65afad4538fc 2024-11-17T08:14:46,577 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/b4ac01c6b1c545fc9e0c65afad4538fc, entries=150, sequenceid=521, filesize=12.0 K 2024-11-17T08:14:46,578 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 988129a9cf65d53c5913e9c54b149fd7 in 1277ms, sequenceid=521, compaction requested=true 2024-11-17T08:14:46,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/443720ad1e2f4e79a63de18962dbb31a, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/36655c9ea9fb457dbf6eabfbf1188845, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/398cdee7f53e424b97c01b7dfdd9021d, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/5da72974eb5b4f81ba9784a9733993a8, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/0f18a73db9b54237ad0c17b8d64ea7c6, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/74aad59bc1504c99b5fe215a41ad0246, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/a4e397e272dc477ab3d60994c677c614, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/975ccd3742eb429ca765486bfc976e87, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/d8734eff0d584e1ba8e99cef0eb670ef, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/d5a8ec86e5c24d548f224cb22c1e5dad, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/4ceffb75b37f4d0d98ba4c0d154eb574, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/649f289786f1491c9f7c6e068a2bd3d3, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/6f1a31bf29884c46a0786d5b15ae25f2, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/3bd90273cca0436d8154c5d476b4bcab, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/49abbb97a36942498593210efe3404e4, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/fb6dc5ab09a94e5a83012f69da13a7e7, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/0c5b18921bf24f0e9ad99aabc802dc28, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/2bfb826ce23c49688e6192eac7223278, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/0453524bc9194b1c9f0efbad6890ed0a, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/a3fb5e16aa6c4536ab46eb483701bd29, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/a58d4d1c00d14e76b105d18f01d31ec3, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/620cf6a8d9064522a06d80c5c816dfb7, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/8730977ca3364008ab85a34d5b038174, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/e861662ad76d470a87f648935c5a27c4, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/8821d88ddcd848c3a5f733fc25e39ec5, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/142298e8120d4658ba51352639d53ab0, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/d3cc78fe3e2a40bdaa6e749d1596bf7c, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/c4ed7bc50210412ea66064aa1beb5642, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/859ec000ff27461da03d0943294c7ce2, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/69b7562a4c6c4575b7a078d5a654056c, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/1be9d4d9bf5341deb6f816c29fbd1b75, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/ada9205f3b6c4bfdb1f6e5408eb81cfb, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/89d7419bbdd648bbb8ed5e71136b783a] to archive 2024-11-17T08:14:46,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T08:14:46,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/443720ad1e2f4e79a63de18962dbb31a to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/443720ad1e2f4e79a63de18962dbb31a 2024-11-17T08:14:46,589 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/36655c9ea9fb457dbf6eabfbf1188845 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/36655c9ea9fb457dbf6eabfbf1188845 2024-11-17T08:14:46,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/398cdee7f53e424b97c01b7dfdd9021d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/398cdee7f53e424b97c01b7dfdd9021d 2024-11-17T08:14:46,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/5da72974eb5b4f81ba9784a9733993a8 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/5da72974eb5b4f81ba9784a9733993a8 2024-11-17T08:14:46,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/0f18a73db9b54237ad0c17b8d64ea7c6 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/0f18a73db9b54237ad0c17b8d64ea7c6 2024-11-17T08:14:46,594 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/74aad59bc1504c99b5fe215a41ad0246 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/74aad59bc1504c99b5fe215a41ad0246 2024-11-17T08:14:46,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/a4e397e272dc477ab3d60994c677c614 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/a4e397e272dc477ab3d60994c677c614 2024-11-17T08:14:46,597 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/975ccd3742eb429ca765486bfc976e87 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/975ccd3742eb429ca765486bfc976e87 2024-11-17T08:14:46,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/d8734eff0d584e1ba8e99cef0eb670ef to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/d8734eff0d584e1ba8e99cef0eb670ef 2024-11-17T08:14:46,600 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/d5a8ec86e5c24d548f224cb22c1e5dad to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/d5a8ec86e5c24d548f224cb22c1e5dad 2024-11-17T08:14:46,601 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/4ceffb75b37f4d0d98ba4c0d154eb574 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/4ceffb75b37f4d0d98ba4c0d154eb574 2024-11-17T08:14:46,602 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/649f289786f1491c9f7c6e068a2bd3d3 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/649f289786f1491c9f7c6e068a2bd3d3 2024-11-17T08:14:46,604 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/6f1a31bf29884c46a0786d5b15ae25f2 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/6f1a31bf29884c46a0786d5b15ae25f2 2024-11-17T08:14:46,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/3bd90273cca0436d8154c5d476b4bcab to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/3bd90273cca0436d8154c5d476b4bcab 2024-11-17T08:14:46,606 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/49abbb97a36942498593210efe3404e4 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/49abbb97a36942498593210efe3404e4 2024-11-17T08:14:46,608 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/fb6dc5ab09a94e5a83012f69da13a7e7 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/fb6dc5ab09a94e5a83012f69da13a7e7 2024-11-17T08:14:46,610 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/0c5b18921bf24f0e9ad99aabc802dc28 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/0c5b18921bf24f0e9ad99aabc802dc28 2024-11-17T08:14:46,611 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/2bfb826ce23c49688e6192eac7223278 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/2bfb826ce23c49688e6192eac7223278 2024-11-17T08:14:46,612 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/0453524bc9194b1c9f0efbad6890ed0a to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/0453524bc9194b1c9f0efbad6890ed0a 2024-11-17T08:14:46,614 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/a3fb5e16aa6c4536ab46eb483701bd29 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/a3fb5e16aa6c4536ab46eb483701bd29 2024-11-17T08:14:46,615 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/a58d4d1c00d14e76b105d18f01d31ec3 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/a58d4d1c00d14e76b105d18f01d31ec3 2024-11-17T08:14:46,617 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/620cf6a8d9064522a06d80c5c816dfb7 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/620cf6a8d9064522a06d80c5c816dfb7 2024-11-17T08:14:46,618 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/8730977ca3364008ab85a34d5b038174 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/8730977ca3364008ab85a34d5b038174 2024-11-17T08:14:46,619 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/e861662ad76d470a87f648935c5a27c4 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/e861662ad76d470a87f648935c5a27c4 2024-11-17T08:14:46,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/8821d88ddcd848c3a5f733fc25e39ec5 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/8821d88ddcd848c3a5f733fc25e39ec5 2024-11-17T08:14:46,622 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/142298e8120d4658ba51352639d53ab0 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/142298e8120d4658ba51352639d53ab0 2024-11-17T08:14:46,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/d3cc78fe3e2a40bdaa6e749d1596bf7c to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/d3cc78fe3e2a40bdaa6e749d1596bf7c 2024-11-17T08:14:46,625 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/c4ed7bc50210412ea66064aa1beb5642 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/c4ed7bc50210412ea66064aa1beb5642 2024-11-17T08:14:46,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/859ec000ff27461da03d0943294c7ce2 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/859ec000ff27461da03d0943294c7ce2 2024-11-17T08:14:46,627 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/69b7562a4c6c4575b7a078d5a654056c to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/69b7562a4c6c4575b7a078d5a654056c 2024-11-17T08:14:46,629 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/1be9d4d9bf5341deb6f816c29fbd1b75 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/1be9d4d9bf5341deb6f816c29fbd1b75 2024-11-17T08:14:46,630 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/ada9205f3b6c4bfdb1f6e5408eb81cfb to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/ada9205f3b6c4bfdb1f6e5408eb81cfb 2024-11-17T08:14:46,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/89d7419bbdd648bbb8ed5e71136b783a to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/89d7419bbdd648bbb8ed5e71136b783a 2024-11-17T08:14:46,644 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/dfaa20c2bce24f978d2763b23894fa7e, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/814237c689d249bfb3a8656fe8e035dc, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/cd74d8753c674dc1aac54f3d44c0ab40, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/c8dad1a3c6d9455986a88d7eabf87b73, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/495a8c51764b4566a9c19a85a8c2ed51, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/dfa2e3b99438442ab7e83478f39e50e1, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/04955baeea824b67b1a78d23d398f3e1, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/703b6f009d6344c199a887c6c72cb683, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/f6dd1472b9db42c3b34a5d0561cb3c44, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/a2d0c489105b45df93ea71a2cf90e99f, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/184b44b7f6934b6eb3fc1212b88ad8c7, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/435f65d529cf44cfbd90396f92080f23, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/d0e90d4700e24a2c9d0899b614c6a021, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/6b0ef1df3f754883b888a6995f4cdd12, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/1c28f33dc48043d2b97a8a0b9a20b594, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/c803d88c0238470a80618a4a3fe7d049, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/89e4c1af56b14cb79c365416a58276d2, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/4a80046370f943398f32f26e2eb8c388, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/2aa124c48f9a4cdb91cf1f3c04cc0c3b, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/0907c705159e4240a36b2d2ef5b14f2d, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/1561353dd0c74cd6b2e5482ca0f964af, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/d9e23c1ad5bc43c0a50af50f6714f75d, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/fced3d05672542888131b06cfe801f46, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/03dab20cc16849989deaf3636193fe52, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/2b9f0596f89c4725a5bce85b8f3464e0, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/437d93837147457aacd73ad951515362, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/e164d7959c6648318b6ab4022f893033, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/4e70e42a263b48f88120277b8640f5b9, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/969ce8656a624fbeb757784da0bc62a8, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/db0c24a7a7574a8593ea56c80b31b3b4, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/d407444192f64469a3c690ed751b2f8f, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/42e5b9b59c724837b3fe0f5d9de4a735, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/e43c8fcaf6744b95a76d810c117ad6d9] to archive 2024-11-17T08:14:46,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T08:14:46,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/dfaa20c2bce24f978d2763b23894fa7e to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/dfaa20c2bce24f978d2763b23894fa7e 2024-11-17T08:14:46,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/814237c689d249bfb3a8656fe8e035dc to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/814237c689d249bfb3a8656fe8e035dc 2024-11-17T08:14:46,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/cd74d8753c674dc1aac54f3d44c0ab40 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/cd74d8753c674dc1aac54f3d44c0ab40 2024-11-17T08:14:46,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/c8dad1a3c6d9455986a88d7eabf87b73 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/c8dad1a3c6d9455986a88d7eabf87b73 2024-11-17T08:14:46,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/495a8c51764b4566a9c19a85a8c2ed51 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/495a8c51764b4566a9c19a85a8c2ed51 2024-11-17T08:14:46,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/dfa2e3b99438442ab7e83478f39e50e1 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/dfa2e3b99438442ab7e83478f39e50e1 2024-11-17T08:14:46,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/04955baeea824b67b1a78d23d398f3e1 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/04955baeea824b67b1a78d23d398f3e1 2024-11-17T08:14:46,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/703b6f009d6344c199a887c6c72cb683 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/703b6f009d6344c199a887c6c72cb683 2024-11-17T08:14:46,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/f6dd1472b9db42c3b34a5d0561cb3c44 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/f6dd1472b9db42c3b34a5d0561cb3c44 2024-11-17T08:14:46,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/a2d0c489105b45df93ea71a2cf90e99f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/a2d0c489105b45df93ea71a2cf90e99f 2024-11-17T08:14:46,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/184b44b7f6934b6eb3fc1212b88ad8c7 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/184b44b7f6934b6eb3fc1212b88ad8c7 2024-11-17T08:14:46,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/435f65d529cf44cfbd90396f92080f23 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/435f65d529cf44cfbd90396f92080f23 2024-11-17T08:14:46,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/d0e90d4700e24a2c9d0899b614c6a021 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/d0e90d4700e24a2c9d0899b614c6a021 2024-11-17T08:14:46,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/6b0ef1df3f754883b888a6995f4cdd12 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/6b0ef1df3f754883b888a6995f4cdd12 2024-11-17T08:14:46,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/1c28f33dc48043d2b97a8a0b9a20b594 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/1c28f33dc48043d2b97a8a0b9a20b594 2024-11-17T08:14:46,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/c803d88c0238470a80618a4a3fe7d049 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/c803d88c0238470a80618a4a3fe7d049 2024-11-17T08:14:46,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/89e4c1af56b14cb79c365416a58276d2 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/89e4c1af56b14cb79c365416a58276d2 2024-11-17T08:14:46,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/4a80046370f943398f32f26e2eb8c388 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/4a80046370f943398f32f26e2eb8c388 2024-11-17T08:14:46,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/2aa124c48f9a4cdb91cf1f3c04cc0c3b to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/2aa124c48f9a4cdb91cf1f3c04cc0c3b 2024-11-17T08:14:46,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/0907c705159e4240a36b2d2ef5b14f2d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/0907c705159e4240a36b2d2ef5b14f2d 2024-11-17T08:14:46,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/1561353dd0c74cd6b2e5482ca0f964af to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/1561353dd0c74cd6b2e5482ca0f964af 2024-11-17T08:14:46,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/d9e23c1ad5bc43c0a50af50f6714f75d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/d9e23c1ad5bc43c0a50af50f6714f75d 2024-11-17T08:14:46,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/fced3d05672542888131b06cfe801f46 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/fced3d05672542888131b06cfe801f46 2024-11-17T08:14:46,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/03dab20cc16849989deaf3636193fe52 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/03dab20cc16849989deaf3636193fe52 2024-11-17T08:14:46,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/2b9f0596f89c4725a5bce85b8f3464e0 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/2b9f0596f89c4725a5bce85b8f3464e0 2024-11-17T08:14:46,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/437d93837147457aacd73ad951515362 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/437d93837147457aacd73ad951515362 2024-11-17T08:14:46,676 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/e164d7959c6648318b6ab4022f893033 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/e164d7959c6648318b6ab4022f893033 2024-11-17T08:14:46,678 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/4e70e42a263b48f88120277b8640f5b9 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/4e70e42a263b48f88120277b8640f5b9 2024-11-17T08:14:46,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/969ce8656a624fbeb757784da0bc62a8 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/969ce8656a624fbeb757784da0bc62a8 2024-11-17T08:14:46,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/db0c24a7a7574a8593ea56c80b31b3b4 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/db0c24a7a7574a8593ea56c80b31b3b4 2024-11-17T08:14:46,681 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/d407444192f64469a3c690ed751b2f8f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/d407444192f64469a3c690ed751b2f8f 2024-11-17T08:14:46,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/42e5b9b59c724837b3fe0f5d9de4a735 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/42e5b9b59c724837b3fe0f5d9de4a735 2024-11-17T08:14:46,683 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/e43c8fcaf6744b95a76d810c117ad6d9 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/e43c8fcaf6744b95a76d810c117ad6d9 2024-11-17T08:14:46,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/62fbc9b549574e5db518e89c9c6b80dc, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/28bdbd75f4d744119b8531e7cee65de4, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/a49cb8bbdd9e4e0b8b29fc9e92722869, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/946a3fcc9ad24a6c9b13d04e76b850ce, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/e0a24c7a91fe43dd88155f4f8c00f247, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/eb1463a46c434432aa1c943de9e6d849, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/b101c1b0311b4c23af2dbf2095f9068b, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/c08eafd1f1c4414fbcf21d1e2d731e14, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/aba29b187c8446ffb1522e68f69ccedc, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/67e9f90ec02c47cdac83666f3f209be1, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/0fd27848b911494190352cee1654b127, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/7c9ef0b184a143f3ae33e85acc91b349, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/0f83f3225d77477088d9edca01cf4a20, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/13f2bbbc40cc47b6972ed2ff16ace1f2, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/f2d1510f85ec465483fe6ed04726f37f, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/822f2d1db44c40e68d6d85eea454e026, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/7503fe94f84243b1950c7a5d864b3359, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/fbb72ae625fa41b29b461b10c6d905ce, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/bcfea8a65043478f876960630da2132a, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/3abd05c7d73b47c0a8f8d925f259977a, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/9911ba0b202040439c40f012edab639f, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/0837e2609b9744d2a82724ce75c994b8, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/e8804de7f67b4ee3949fff476d6def36, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/fef04c307851497d8cb27bb1df941429, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/8f9545518c1d4b728fea8cd92bc2bd70, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/e9fce002f9f6487295045f199b6696f2, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/9b76b0eae09c476dbb569bd7967f6c6d, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/61d2546038374eea86d5f384cc74d866, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/7ec8e6a2957a42a2931dab4a5e247ade, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/9f914f2428414958b001b168b5b3d2eb, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/815448df7e8c46c7816105248ab1cce1, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/5aa9f30265ac4b6086e0262450e5f58e, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/a8169c3c34434756bba3457700e70c80] to archive 2024-11-17T08:14:46,686 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T08:14:46,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/62fbc9b549574e5db518e89c9c6b80dc to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/62fbc9b549574e5db518e89c9c6b80dc 2024-11-17T08:14:46,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/28bdbd75f4d744119b8531e7cee65de4 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/28bdbd75f4d744119b8531e7cee65de4 2024-11-17T08:14:46,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/a49cb8bbdd9e4e0b8b29fc9e92722869 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/a49cb8bbdd9e4e0b8b29fc9e92722869 2024-11-17T08:14:46,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/946a3fcc9ad24a6c9b13d04e76b850ce to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/946a3fcc9ad24a6c9b13d04e76b850ce 2024-11-17T08:14:46,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/e0a24c7a91fe43dd88155f4f8c00f247 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/e0a24c7a91fe43dd88155f4f8c00f247 2024-11-17T08:14:46,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/eb1463a46c434432aa1c943de9e6d849 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/eb1463a46c434432aa1c943de9e6d849 2024-11-17T08:14:46,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/b101c1b0311b4c23af2dbf2095f9068b to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/b101c1b0311b4c23af2dbf2095f9068b 2024-11-17T08:14:46,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/c08eafd1f1c4414fbcf21d1e2d731e14 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/c08eafd1f1c4414fbcf21d1e2d731e14 2024-11-17T08:14:46,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/aba29b187c8446ffb1522e68f69ccedc to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/aba29b187c8446ffb1522e68f69ccedc 2024-11-17T08:14:46,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/67e9f90ec02c47cdac83666f3f209be1 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/67e9f90ec02c47cdac83666f3f209be1 2024-11-17T08:14:46,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/0fd27848b911494190352cee1654b127 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/0fd27848b911494190352cee1654b127 2024-11-17T08:14:46,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/7c9ef0b184a143f3ae33e85acc91b349 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/7c9ef0b184a143f3ae33e85acc91b349 2024-11-17T08:14:46,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/0f83f3225d77477088d9edca01cf4a20 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/0f83f3225d77477088d9edca01cf4a20 2024-11-17T08:14:46,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/13f2bbbc40cc47b6972ed2ff16ace1f2 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/13f2bbbc40cc47b6972ed2ff16ace1f2 2024-11-17T08:14:46,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/f2d1510f85ec465483fe6ed04726f37f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/f2d1510f85ec465483fe6ed04726f37f 2024-11-17T08:14:46,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/822f2d1db44c40e68d6d85eea454e026 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/822f2d1db44c40e68d6d85eea454e026 2024-11-17T08:14:46,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/7503fe94f84243b1950c7a5d864b3359 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/7503fe94f84243b1950c7a5d864b3359 2024-11-17T08:14:46,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/fbb72ae625fa41b29b461b10c6d905ce to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/fbb72ae625fa41b29b461b10c6d905ce 2024-11-17T08:14:46,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/bcfea8a65043478f876960630da2132a to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/bcfea8a65043478f876960630da2132a 2024-11-17T08:14:46,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/3abd05c7d73b47c0a8f8d925f259977a to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/3abd05c7d73b47c0a8f8d925f259977a 2024-11-17T08:14:46,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/9911ba0b202040439c40f012edab639f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/9911ba0b202040439c40f012edab639f 2024-11-17T08:14:46,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/0837e2609b9744d2a82724ce75c994b8 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/0837e2609b9744d2a82724ce75c994b8 2024-11-17T08:14:46,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/e8804de7f67b4ee3949fff476d6def36 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/e8804de7f67b4ee3949fff476d6def36 2024-11-17T08:14:46,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/fef04c307851497d8cb27bb1df941429 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/fef04c307851497d8cb27bb1df941429 2024-11-17T08:14:46,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/8f9545518c1d4b728fea8cd92bc2bd70 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/8f9545518c1d4b728fea8cd92bc2bd70 2024-11-17T08:14:46,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/e9fce002f9f6487295045f199b6696f2 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/e9fce002f9f6487295045f199b6696f2 2024-11-17T08:14:46,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/9b76b0eae09c476dbb569bd7967f6c6d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/9b76b0eae09c476dbb569bd7967f6c6d 2024-11-17T08:14:46,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/61d2546038374eea86d5f384cc74d866 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/61d2546038374eea86d5f384cc74d866 2024-11-17T08:14:46,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/7ec8e6a2957a42a2931dab4a5e247ade to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/7ec8e6a2957a42a2931dab4a5e247ade 2024-11-17T08:14:46,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/9f914f2428414958b001b168b5b3d2eb to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/9f914f2428414958b001b168b5b3d2eb 2024-11-17T08:14:46,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/815448df7e8c46c7816105248ab1cce1 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/815448df7e8c46c7816105248ab1cce1 2024-11-17T08:14:46,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/5aa9f30265ac4b6086e0262450e5f58e to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/5aa9f30265ac4b6086e0262450e5f58e 2024-11-17T08:14:46,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/a8169c3c34434756bba3457700e70c80 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/a8169c3c34434756bba3457700e70c80 2024-11-17T08:14:46,727 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/recovered.edits/524.seqid, newMaxSeqId=524, maxSeqId=1 2024-11-17T08:14:46,728 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. 2024-11-17T08:14:46,728 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for 988129a9cf65d53c5913e9c54b149fd7: Waiting for close lock at 1731831285301Running coprocessor pre-close hooks at 1731831285301Disabling compacts and flushes for region at 1731831285301Disabling writes for close at 1731831285301Obtaining lock to block concurrent updates at 1731831285302 (+1 ms)Preparing flush snapshotting stores in 988129a9cf65d53c5913e9c54b149fd7 at 1731831285302Finished memstore snapshotting TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731831285303 (+1 ms)Flushing stores of TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7. at 1731831285303Flushing 988129a9cf65d53c5913e9c54b149fd7/A: creating writer at 1731831285303Flushing 988129a9cf65d53c5913e9c54b149fd7/A: appending metadata at 1731831285307 (+4 ms)Flushing 988129a9cf65d53c5913e9c54b149fd7/A: closing flushed file at 1731831285307Flushing 988129a9cf65d53c5913e9c54b149fd7/B: creating writer at 1731831285720 (+413 ms)Flushing 988129a9cf65d53c5913e9c54b149fd7/B: appending metadata at 1731831285723 (+3 ms)Flushing 988129a9cf65d53c5913e9c54b149fd7/B: closing flushed file at 1731831285723Flushing 988129a9cf65d53c5913e9c54b149fd7/C: creating writer at 1731831286137 (+414 ms)Flushing 988129a9cf65d53c5913e9c54b149fd7/C: appending metadata at 1731831286141 (+4 ms)Flushing 988129a9cf65d53c5913e9c54b149fd7/C: closing flushed file at 1731831286141Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27e38225: reopening flushed file at 1731831286557 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1cee3c13: reopening flushed file at 1731831286565 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a4c1293: reopening flushed file at 1731831286572 (+7 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 988129a9cf65d53c5913e9c54b149fd7 in 1277ms, sequenceid=521, compaction requested=true at 1731831286578 (+6 ms)Writing region close event to WAL at 1731831286723 (+145 ms)Running coprocessor post-close hooks at 1731831286728 (+5 ms)Closed at 1731831286728 2024-11-17T08:14:46,731 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed 988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:46,731 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=988129a9cf65d53c5913e9c54b149fd7, regionState=CLOSED 2024-11-17T08:14:46,734 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 because future has completed 2024-11-17T08:14:46,738 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-17T08:14:46,738 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure 988129a9cf65d53c5913e9c54b149fd7, server=e3d0adb87c01,40631,1731831247805 in 1.5910 sec 2024-11-17T08:14:46,739 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-17T08:14:46,740 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=988129a9cf65d53c5913e9c54b149fd7, UNASSIGN in 1.6000 sec 2024-11-17T08:14:46,743 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-17T08:14:46,743 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6070 sec 2024-11-17T08:14:46,745 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731831286745"}]},"ts":"1731831286745"} 2024-11-17T08:14:46,747 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T08:14:46,747 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T08:14:46,750 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6220 sec 2024-11-17T08:14:47,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-17T08:14:47,285 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:14:47,292 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-17T08:14:47,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T08:14:47,298 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T08:14:47,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-17T08:14:47,300 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=33, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T08:14:47,306 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:47,310 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A, FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B, FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C, FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/recovered.edits] 2024-11-17T08:14:47,313 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/73df514594ae4742ab560ba3b9f4c4c6 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/73df514594ae4742ab560ba3b9f4c4c6 2024-11-17T08:14:47,315 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/ada1d0e952c844b9b35611376077bd04 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/ada1d0e952c844b9b35611376077bd04 2024-11-17T08:14:47,316 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/f3a061e8baad4ace871376eda9ce43b3 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/A/f3a061e8baad4ace871376eda9ce43b3 2024-11-17T08:14:47,318 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/29a536f1e6d44b83962906ec5be86020 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/29a536f1e6d44b83962906ec5be86020 2024-11-17T08:14:47,319 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/97d392b309bf43e08e065c6e4efd04e3 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/97d392b309bf43e08e065c6e4efd04e3 2024-11-17T08:14:47,320 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/fc78dd906ceb492c9abca10f8d68b02b to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/B/fc78dd906ceb492c9abca10f8d68b02b 2024-11-17T08:14:47,323 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/0351b21016194a80aec032ca13a2aa3a to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/0351b21016194a80aec032ca13a2aa3a 2024-11-17T08:14:47,324 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/b4ac01c6b1c545fc9e0c65afad4538fc to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/b4ac01c6b1c545fc9e0c65afad4538fc 2024-11-17T08:14:47,327 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/db5e377a92a4497f86d595365acbbc47 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/C/db5e377a92a4497f86d595365acbbc47 2024-11-17T08:14:47,327 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-17T08:14:47,330 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/recovered.edits/524.seqid to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7/recovered.edits/524.seqid 2024-11-17T08:14:47,331 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/988129a9cf65d53c5913e9c54b149fd7 2024-11-17T08:14:47,331 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T08:14:47,336 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=33, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T08:14:47,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-17T08:14:47,346 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T08:14:47,373 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T08:14:47,375 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=33, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T08:14:47,375 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T08:14:47,375 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731831287375"}]},"ts":"9223372036854775807"} 2024-11-17T08:14:47,379 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T08:14:47,379 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 988129a9cf65d53c5913e9c54b149fd7, NAME => 'TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T08:14:47,379 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T08:14:47,380 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731831287379"}]},"ts":"9223372036854775807"} 2024-11-17T08:14:47,383 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T08:14:47,384 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=33, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T08:14:47,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 91 msec 2024-11-17T08:14:47,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-17T08:14:47,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T08:14:47,404 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:14:47,415 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=248 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;e3d0adb87c01:40631-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-306999293_22 at /127.0.0.1:43694 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=288 (was 230) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2293 (was 2953) 2024-11-17T08:14:47,426 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=288, ProcessCount=11, AvailableMemoryMB=2289 2024-11-17T08:14:47,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T08:14:47,428 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T08:14:47,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T08:14:47,431 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T08:14:47,431 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:14:47,431 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 34 2024-11-17T08:14:47,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-17T08:14:47,433 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T08:14:47,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741945_1121 (size=1061) 2024-11-17T08:14:47,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-17T08:14:47,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-17T08:14:47,846 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85 2024-11-17T08:14:47,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741946_1122 (size=53) 2024-11-17T08:14:48,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-17T08:14:48,257 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T08:14:48,258 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e8e9eb0dab895bf5f9f5249707e9712d, disabling compactions & flushes 2024-11-17T08:14:48,258 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:48,258 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:48,259 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. after waiting 0 ms 2024-11-17T08:14:48,259 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:48,259 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:48,259 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e8e9eb0dab895bf5f9f5249707e9712d: Waiting for close lock at 1731831288258Disabling compacts and flushes for region at 1731831288258Disabling writes for close at 1731831288259 (+1 ms)Writing region close event to WAL at 1731831288259Closed at 1731831288259 2024-11-17T08:14:48,262 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T08:14:48,262 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731831288262"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731831288262"}]},"ts":"1731831288262"} 2024-11-17T08:14:48,266 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T08:14:48,268 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T08:14:48,268 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731831288268"}]},"ts":"1731831288268"} 2024-11-17T08:14:48,270 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T08:14:48,270 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e8e9eb0dab895bf5f9f5249707e9712d, ASSIGN}] 2024-11-17T08:14:48,271 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e8e9eb0dab895bf5f9f5249707e9712d, ASSIGN 2024-11-17T08:14:48,272 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e8e9eb0dab895bf5f9f5249707e9712d, ASSIGN; state=OFFLINE, location=e3d0adb87c01,40631,1731831247805; forceNewPlan=false, retain=false 2024-11-17T08:14:48,424 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=e8e9eb0dab895bf5f9f5249707e9712d, regionState=OPENING, regionLocation=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:48,428 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e8e9eb0dab895bf5f9f5249707e9712d, ASSIGN because future has completed 2024-11-17T08:14:48,429 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805}] 2024-11-17T08:14:48,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-17T08:14:48,590 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:48,591 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} 2024-11-17T08:14:48,591 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:48,592 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T08:14:48,592 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:48,592 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:48,595 INFO [StoreOpener-e8e9eb0dab895bf5f9f5249707e9712d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:48,596 INFO [StoreOpener-e8e9eb0dab895bf5f9f5249707e9712d-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T08:14:48,597 INFO [StoreOpener-e8e9eb0dab895bf5f9f5249707e9712d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e8e9eb0dab895bf5f9f5249707e9712d columnFamilyName A 2024-11-17T08:14:48,598 DEBUG [StoreOpener-e8e9eb0dab895bf5f9f5249707e9712d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:14:48,598 INFO [StoreOpener-e8e9eb0dab895bf5f9f5249707e9712d-1 {}] regionserver.HStore(327): Store=e8e9eb0dab895bf5f9f5249707e9712d/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T08:14:48,598 INFO [StoreOpener-e8e9eb0dab895bf5f9f5249707e9712d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:48,600 INFO [StoreOpener-e8e9eb0dab895bf5f9f5249707e9712d-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T08:14:48,600 INFO [StoreOpener-e8e9eb0dab895bf5f9f5249707e9712d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e8e9eb0dab895bf5f9f5249707e9712d columnFamilyName B 2024-11-17T08:14:48,600 DEBUG [StoreOpener-e8e9eb0dab895bf5f9f5249707e9712d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:14:48,600 INFO [StoreOpener-e8e9eb0dab895bf5f9f5249707e9712d-1 {}] regionserver.HStore(327): Store=e8e9eb0dab895bf5f9f5249707e9712d/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T08:14:48,601 INFO [StoreOpener-e8e9eb0dab895bf5f9f5249707e9712d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:48,602 INFO [StoreOpener-e8e9eb0dab895bf5f9f5249707e9712d-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T08:14:48,602 INFO [StoreOpener-e8e9eb0dab895bf5f9f5249707e9712d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e8e9eb0dab895bf5f9f5249707e9712d columnFamilyName C 2024-11-17T08:14:48,602 DEBUG [StoreOpener-e8e9eb0dab895bf5f9f5249707e9712d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:14:48,602 INFO [StoreOpener-e8e9eb0dab895bf5f9f5249707e9712d-1 {}] regionserver.HStore(327): Store=e8e9eb0dab895bf5f9f5249707e9712d/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T08:14:48,602 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:48,602 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:48,603 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:48,603 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:48,604 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:48,604 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:48,605 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T08:14:48,606 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:48,608 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T08:14:48,608 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened e8e9eb0dab895bf5f9f5249707e9712d; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68774157, jitterRate=0.024814799427986145}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T08:14:48,608 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:48,609 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for e8e9eb0dab895bf5f9f5249707e9712d: Running coprocessor pre-open hook at 1731831288592Writing region info on filesystem at 1731831288592Initializing all the Stores at 1731831288594 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731831288594Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731831288594Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731831288595 (+1 ms)Cleaning up temporary data from old regions at 1731831288604 (+9 ms)Running coprocessor post-open hooks at 1731831288608 (+4 ms)Region opened successfully at 1731831288609 (+1 ms) 2024-11-17T08:14:48,609 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., pid=36, masterSystemTime=1731831288582 2024-11-17T08:14:48,611 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:48,611 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:48,612 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=e8e9eb0dab895bf5f9f5249707e9712d, regionState=OPEN, openSeqNum=2, regionLocation=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:48,614 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 because future has completed 2024-11-17T08:14:48,616 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-17T08:14:48,617 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 in 186 msec 2024-11-17T08:14:48,618 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-17T08:14:48,618 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e8e9eb0dab895bf5f9f5249707e9712d, ASSIGN in 347 msec 2024-11-17T08:14:48,619 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T08:14:48,619 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731831288619"}]},"ts":"1731831288619"} 2024-11-17T08:14:48,621 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T08:14:48,622 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T08:14:48,623 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1940 sec 2024-11-17T08:14:49,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-17T08:14:49,584 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:14:49,586 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2697ccde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:49,586 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:14:49,586 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:14:49,588 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:14:49,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:14:49,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:14:49,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e595508, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:49,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:14:49,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:14:49,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:14:49,592 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:14:49,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b7e2b23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:49,594 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:14:49,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:14:49,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:14:49,597 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:14:49,599 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T08:14:49,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is e3d0adb87c01,45089,1731831246985 2024-11-17T08:14:49,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@557415e3 2024-11-17T08:14:49,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T08:14:49,601 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T08:14:49,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T08:14:49,606 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T08:14:49,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-17T08:14:49,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741947_1123 (size=1097) 2024-11-17T08:14:50,027 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-17T08:14:50,027 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-17T08:14:50,033 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T08:14:50,040 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e8e9eb0dab895bf5f9f5249707e9712d, REOPEN/MOVE}] 2024-11-17T08:14:50,041 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e8e9eb0dab895bf5f9f5249707e9712d, REOPEN/MOVE 2024-11-17T08:14:50,042 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=e8e9eb0dab895bf5f9f5249707e9712d, regionState=CLOSING, regionLocation=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:50,043 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e8e9eb0dab895bf5f9f5249707e9712d, REOPEN/MOVE because future has completed 2024-11-17T08:14:50,044 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T08:14:50,044 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805}] 2024-11-17T08:14:50,197 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:50,197 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T08:14:50,197 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing e8e9eb0dab895bf5f9f5249707e9712d, disabling compactions & flushes 2024-11-17T08:14:50,197 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:50,197 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:50,197 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. after waiting 0 ms 2024-11-17T08:14:50,197 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:50,260 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T08:14:50,262 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:50,262 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for e8e9eb0dab895bf5f9f5249707e9712d: Waiting for close lock at 1731831290197Running coprocessor pre-close hooks at 1731831290197Disabling compacts and flushes for region at 1731831290197Disabling writes for close at 1731831290197Writing region close event to WAL at 1731831290197Running coprocessor post-close hooks at 1731831290262 (+65 ms)Closed at 1731831290262 2024-11-17T08:14:50,262 WARN [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionServer(3299): Not adding moved region record: e8e9eb0dab895bf5f9f5249707e9712d to self. 2024-11-17T08:14:50,267 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:50,268 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=e8e9eb0dab895bf5f9f5249707e9712d, regionState=CLOSED 2024-11-17T08:14:50,270 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 because future has completed 2024-11-17T08:14:50,275 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-17T08:14:50,275 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; CloseRegionProcedure e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 in 228 msec 2024-11-17T08:14:50,276 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e8e9eb0dab895bf5f9f5249707e9712d, REOPEN/MOVE; state=CLOSED, location=e3d0adb87c01,40631,1731831247805; forceNewPlan=false, retain=true 2024-11-17T08:14:50,427 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=e8e9eb0dab895bf5f9f5249707e9712d, regionState=OPENING, regionLocation=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:50,432 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e8e9eb0dab895bf5f9f5249707e9712d, REOPEN/MOVE because future has completed 2024-11-17T08:14:50,432 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805}] 2024-11-17T08:14:50,593 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:50,593 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} 2024-11-17T08:14:50,594 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:50,594 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T08:14:50,594 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:50,594 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:50,599 INFO [StoreOpener-e8e9eb0dab895bf5f9f5249707e9712d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:50,600 INFO [StoreOpener-e8e9eb0dab895bf5f9f5249707e9712d-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T08:14:50,608 INFO [StoreOpener-e8e9eb0dab895bf5f9f5249707e9712d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e8e9eb0dab895bf5f9f5249707e9712d columnFamilyName A 2024-11-17T08:14:50,611 DEBUG [StoreOpener-e8e9eb0dab895bf5f9f5249707e9712d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:14:50,612 INFO [StoreOpener-e8e9eb0dab895bf5f9f5249707e9712d-1 {}] regionserver.HStore(327): Store=e8e9eb0dab895bf5f9f5249707e9712d/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T08:14:50,612 INFO [StoreOpener-e8e9eb0dab895bf5f9f5249707e9712d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:50,613 INFO [StoreOpener-e8e9eb0dab895bf5f9f5249707e9712d-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T08:14:50,613 INFO [StoreOpener-e8e9eb0dab895bf5f9f5249707e9712d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e8e9eb0dab895bf5f9f5249707e9712d columnFamilyName B 2024-11-17T08:14:50,613 DEBUG [StoreOpener-e8e9eb0dab895bf5f9f5249707e9712d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:14:50,614 INFO [StoreOpener-e8e9eb0dab895bf5f9f5249707e9712d-1 {}] regionserver.HStore(327): Store=e8e9eb0dab895bf5f9f5249707e9712d/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T08:14:50,614 INFO [StoreOpener-e8e9eb0dab895bf5f9f5249707e9712d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:50,614 INFO [StoreOpener-e8e9eb0dab895bf5f9f5249707e9712d-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T08:14:50,614 INFO [StoreOpener-e8e9eb0dab895bf5f9f5249707e9712d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e8e9eb0dab895bf5f9f5249707e9712d columnFamilyName C 2024-11-17T08:14:50,614 DEBUG [StoreOpener-e8e9eb0dab895bf5f9f5249707e9712d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:14:50,615 INFO [StoreOpener-e8e9eb0dab895bf5f9f5249707e9712d-1 {}] regionserver.HStore(327): Store=e8e9eb0dab895bf5f9f5249707e9712d/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T08:14:50,615 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:50,615 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:50,615 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:50,616 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:50,617 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:50,617 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:50,618 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T08:14:50,619 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:50,620 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened e8e9eb0dab895bf5f9f5249707e9712d; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64338564, jitterRate=-0.04128068685531616}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T08:14:50,620 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:50,622 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for e8e9eb0dab895bf5f9f5249707e9712d: Running coprocessor pre-open hook at 1731831290595Writing region info on filesystem at 1731831290595Initializing all the Stores at 1731831290596 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731831290597 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731831290597Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731831290597Cleaning up temporary data from old regions at 1731831290617 (+20 ms)Running coprocessor post-open hooks at 1731831290620 (+3 ms)Region opened successfully at 1731831290622 (+2 ms) 2024-11-17T08:14:50,623 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., pid=41, masterSystemTime=1731831290585 2024-11-17T08:14:50,624 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:50,624 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:50,625 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=e8e9eb0dab895bf5f9f5249707e9712d, regionState=OPEN, openSeqNum=5, regionLocation=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:50,626 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 because future has completed 2024-11-17T08:14:50,629 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=39 2024-11-17T08:14:50,629 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 in 195 msec 2024-11-17T08:14:50,631 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-17T08:14:50,631 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e8e9eb0dab895bf5f9f5249707e9712d, REOPEN/MOVE in 589 msec 2024-11-17T08:14:50,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-17T08:14:50,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 600 msec 2024-11-17T08:14:50,638 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 1.0280 sec 2024-11-17T08:14:50,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-11-17T08:14:50,640 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:14:50,641 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:50,641 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:14:50,641 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:14:50,642 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:14:50,642 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:14:50,642 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:14:50,642 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32e91f17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:50,642 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:14:50,642 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:14:50,643 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:14:50,643 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:14:50,644 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cafc278, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:50,645 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35409f72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:50,645 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:14:50,645 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:14:50,646 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:14:50,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:14:50,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:14:50,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d098d10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:50,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:14:50,647 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:14:50,647 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:14:50,648 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50828, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:14:50,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ab4f03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:50,649 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7944f9cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:50,649 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:14:50,649 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:14:50,650 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:14:50,650 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:14:50,650 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:14:50,650 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6bd813, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:50,650 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:14:50,651 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:14:50,651 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:14:50,652 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50832, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:14:50,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e3ebdec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:50,653 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38720fa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:50,653 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:14:50,653 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:14:50,654 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:14:50,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:14:50,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:14:50,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ebf8f4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:50,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:14:50,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:14:50,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:14:50,656 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:14:50,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7033ad03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:50,657 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c77b203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:50,657 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:14:50,657 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:14:50,658 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:14:50,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:14:50,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:14:50,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@618b87ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:50,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:14:50,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:14:50,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:14:50,660 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:14:50,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43ef2f4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:50,661 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3f6eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:50,662 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:14:50,662 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:14:50,662 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:14:50,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:14:50,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:14:50,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@426c3354, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:50,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:14:50,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:14:50,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:14:50,664 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50894, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:14:50,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28ea510, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:50,665 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40353459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:50,665 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:14:50,665 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:14:50,666 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:14:50,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:14:50,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:14:50,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@293659ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:50,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:14:50,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:14:50,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:14:50,667 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:14:50,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3af31bce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:50,669 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@160383af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:50,669 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:14:50,669 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:14:50,670 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:14:50,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:14:50,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:14:50,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59e58e17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:50,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:14:50,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:14:50,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:14:50,671 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:14:50,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72737, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:50,672 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@123a694c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:50,672 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:14:50,672 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:14:50,673 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:14:50,673 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:14:50,673 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:14:50,674 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26cbe636, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:50,674 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:14:50,674 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:14:50,674 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:14:50,675 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50930, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:14:50,675 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f5286fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:14:50,676 DEBUG [Thread-631 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:14:50,676 DEBUG [Thread-640 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:14:50,676 DEBUG [Thread-627 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:14:50,676 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:14:50,677 DEBUG [Thread-636 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:14:50,677 DEBUG [Thread-634 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:14:50,677 DEBUG [Thread-623 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:14:50,677 DEBUG [Thread-629 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:14:50,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:14:50,677 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:14:50,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:14:50,678 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:14:50,678 DEBUG [Thread-634 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:14:50,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:14:50,678 DEBUG [Thread-636 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:14:50,678 DEBUG [Thread-636 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:14:50,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-17T08:14:50,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:14:50,678 DEBUG [Thread-625 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:14:50,679 DEBUG [Thread-638 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:14:50,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:14:50,679 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:14:50,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:14:50,680 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44050, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:14:50,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:14:50,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T08:14:50,680 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44080, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:14:50,680 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:14:50,681 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44064, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:14:50,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:14:50,681 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44100, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:14:50,681 DEBUG [Thread-634 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:14:50,682 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:14:50,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:14:50,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:14:50,682 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44112, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:14:50,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:14:50,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:14:50,683 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:14:50,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5] 2024-11-17T08:14:50,683 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:14:50,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5] 2024-11-17T08:14:50,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5] 2024-11-17T08:14:50,685 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44144, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:14:50,685 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44138, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:14:50,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5] 2024-11-17T08:14:50,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5] 2024-11-17T08:14:50,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5] 2024-11-17T08:14:50,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5] 2024-11-17T08:14:50,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5] 2024-11-17T08:14:50,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5] 2024-11-17T08:14:50,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:50,725 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e8e9eb0dab895bf5f9f5249707e9712d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T08:14:50,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=A 2024-11-17T08:14:50,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:50,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=B 2024-11-17T08:14:50,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:50,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=C 2024-11-17T08:14:50,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:50,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411176ffb0b0fc3bb49c9b15f33a10dea712d_e8e9eb0dab895bf5f9f5249707e9712d is 50, key is test_row_0/A:col10/1731831290695/Put/seqid=0 2024-11-17T08:14:50,777 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731831290751, max timestamp=1731831290771Num uniques 859; 2024-11-17T08:14:50,777 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731831290751, max timestamp=1731831290771Num uniques 859; 2024-11-17T08:14:50,777 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T08:14:50,777 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731831290751, max timestamp=1731831290771Num uniques 859; 2024-11-17T08:14:50,777 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T08:14:50,777 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T08:14:50,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T08:14:50,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741948_1124 (size=12154) 2024-11-17T08:14:50,795 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:14:50,801 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411176ffb0b0fc3bb49c9b15f33a10dea712d_e8e9eb0dab895bf5f9f5249707e9712d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176ffb0b0fc3bb49c9b15f33a10dea712d_e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:50,804 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/d9356a37f801441e84c1577c0e921067, store: [table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:14:50,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/d9356a37f801441e84c1577c0e921067 is 175, key is test_row_0/A:col10/1731831290695/Put/seqid=0 2024-11-17T08:14:50,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:50,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831350810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:50,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:50,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831350813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:50,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:50,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:50,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831350816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:50,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:50,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:50,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:50,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:50,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:50,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:50,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:50,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831350820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:50,829 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:50,829 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:50,829 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:50,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:50,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831350823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:50,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:50,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:50,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:50,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T08:14:50,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:50,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. as already flushing 2024-11-17T08:14:50,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:50,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:50,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:50,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:50,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741949_1125 (size=30955) 2024-11-17T08:14:50,857 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/d9356a37f801441e84c1577c0e921067 2024-11-17T08:14:50,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/d33819c99464452cbdf3e97c74c65e94 is 50, key is test_row_0/B:col10/1731831290695/Put/seqid=0 2024-11-17T08:14:50,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741950_1126 (size=12001) 2024-11-17T08:14:50,896 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/d33819c99464452cbdf3e97c74c65e94 2024-11-17T08:14:50,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:50,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831350948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:50,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/bfae669b183f49cea2b6fb32e037b788 is 50, key is test_row_0/C:col10/1731831290695/Put/seqid=0 2024-11-17T08:14:50,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:50,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831350948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:50,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:50,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:50,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:50,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:50,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831350948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:50,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:50,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:50,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:50,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:50,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831350949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:50,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:50,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:50,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:50,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:50,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831350949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:50,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:50,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:50,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:50,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:50,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:50,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:50,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741951_1127 (size=12001) 2024-11-17T08:14:50,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/bfae669b183f49cea2b6fb32e037b788 2024-11-17T08:14:50,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/d9356a37f801441e84c1577c0e921067 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/d9356a37f801441e84c1577c0e921067 2024-11-17T08:14:50,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T08:14:50,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:50,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. as already flushing 2024-11-17T08:14:50,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:50,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:50,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:50,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:50,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T08:14:51,005 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/d9356a37f801441e84c1577c0e921067, entries=150, sequenceid=14, filesize=30.2 K 2024-11-17T08:14:51,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/d33819c99464452cbdf3e97c74c65e94 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/d33819c99464452cbdf3e97c74c65e94 2024-11-17T08:14:51,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/d33819c99464452cbdf3e97c74c65e94, entries=150, sequenceid=14, filesize=11.7 K 2024-11-17T08:14:51,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/bfae669b183f49cea2b6fb32e037b788 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/bfae669b183f49cea2b6fb32e037b788 2024-11-17T08:14:51,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/bfae669b183f49cea2b6fb32e037b788, entries=150, sequenceid=14, filesize=11.7 K 2024-11-17T08:14:51,023 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for e8e9eb0dab895bf5f9f5249707e9712d in 298ms, sequenceid=14, compaction requested=false 2024-11-17T08:14:51,023 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-17T08:14:51,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e8e9eb0dab895bf5f9f5249707e9712d: 2024-11-17T08:14:51,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T08:14:51,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:51,145 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing e8e9eb0dab895bf5f9f5249707e9712d 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-17T08:14:51,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=A 2024-11-17T08:14:51,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:14:51,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=B 2024-11-17T08:14:51,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:14:51,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=C 2024-11-17T08:14:51,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:14:51,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:51,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. as already flushing 2024-11-17T08:14:51,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117985bef6b1fd64212b3f9e46f3d06953e_e8e9eb0dab895bf5f9f5249707e9712d is 50, key is test_row_0/A:col10/1731831290813/Put/seqid=0 2024-11-17T08:14:51,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741952_1128 (size=12154) 2024-11-17T08:14:51,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:14:51,207 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117985bef6b1fd64212b3f9e46f3d06953e_e8e9eb0dab895bf5f9f5249707e9712d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117985bef6b1fd64212b3f9e46f3d06953e_e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:51,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/035678266982435f8d6ba4d2579d6ee2, store: [table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:14:51,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/035678266982435f8d6ba4d2579d6ee2 is 175, key is test_row_0/A:col10/1731831290813/Put/seqid=0 2024-11-17T08:14:51,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:51,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831351186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:51,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:51,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:51,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:51,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:51,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831351194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:51,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:51,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:51,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:51,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741953_1129 (size=30955) 2024-11-17T08:14:51,222 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/035678266982435f8d6ba4d2579d6ee2 2024-11-17T08:14:51,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/52920993ae8c409b83231c9bab4f8da5 is 50, key is test_row_0/B:col10/1731831290813/Put/seqid=0 2024-11-17T08:14:51,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:51,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831351215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:51,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:51,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:51,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:51,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:51,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:51,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831351218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:51,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831351221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:51,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:51,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:51,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:51,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:51,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:51,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:51,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741954_1130 (size=12001) 2024-11-17T08:14:51,264 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/52920993ae8c409b83231c9bab4f8da5 2024-11-17T08:14:51,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/5580bc6eec3a41d497d77687f580acdc is 50, key is test_row_0/C:col10/1731831290813/Put/seqid=0 2024-11-17T08:14:51,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741955_1131 (size=12001) 2024-11-17T08:14:51,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T08:14:51,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831351324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:51,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:51,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831351325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:51,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:51,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:51,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:51,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:51,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:51,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:51,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831351344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:51,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:51,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:51,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:51,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:51,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831351345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:51,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:51,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:51,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:51,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:51,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831351345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:51,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:51,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:51,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:51,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831351534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:51,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:51,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:51,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:51,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831351534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:51,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:51,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:51,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:51,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:51,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831351553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831351554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:51,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:51,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:51,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:51,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:51,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831351555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:51,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:51,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:51,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:51,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:51,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:51,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:51,695 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/5580bc6eec3a41d497d77687f580acdc 2024-11-17T08:14:51,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/035678266982435f8d6ba4d2579d6ee2 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/035678266982435f8d6ba4d2579d6ee2 2024-11-17T08:14:51,711 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/035678266982435f8d6ba4d2579d6ee2, entries=150, sequenceid=51, filesize=30.2 K 2024-11-17T08:14:51,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/52920993ae8c409b83231c9bab4f8da5 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/52920993ae8c409b83231c9bab4f8da5 2024-11-17T08:14:51,726 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/52920993ae8c409b83231c9bab4f8da5, entries=150, sequenceid=51, filesize=11.7 K 2024-11-17T08:14:51,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/5580bc6eec3a41d497d77687f580acdc as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/5580bc6eec3a41d497d77687f580acdc 2024-11-17T08:14:51,738 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/5580bc6eec3a41d497d77687f580acdc, entries=150, sequenceid=51, filesize=11.7 K 2024-11-17T08:14:51,739 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=53.67 KB/54960 for e8e9eb0dab895bf5f9f5249707e9712d in 595ms, sequenceid=51, compaction requested=false 2024-11-17T08:14:51,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for e8e9eb0dab895bf5f9f5249707e9712d: 2024-11-17T08:14:51,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:51,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-17T08:14:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-17T08:14:51,746 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-17T08:14:51,746 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0610 sec 2024-11-17T08:14:51,749 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.0700 sec 2024-11-17T08:14:51,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T08:14:51,814 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:14:51,815 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:14:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-17T08:14:51,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T08:14:51,817 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:14:51,817 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:14:51,818 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:14:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:51,848 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e8e9eb0dab895bf5f9f5249707e9712d 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-17T08:14:51,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=A 2024-11-17T08:14:51,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:51,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=B 2024-11-17T08:14:51,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:51,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=C 2024-11-17T08:14:51,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:51,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d52e008b6b4340caacb397e2cb3f5b39_e8e9eb0dab895bf5f9f5249707e9712d is 50, key is test_row_0/A:col10/1731831291846/Put/seqid=0 2024-11-17T08:14:51,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741956_1132 (size=14594) 2024-11-17T08:14:51,881 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:14:51,889 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d52e008b6b4340caacb397e2cb3f5b39_e8e9eb0dab895bf5f9f5249707e9712d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d52e008b6b4340caacb397e2cb3f5b39_e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:51,890 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/8bbbc2dea6984a3893124f53ecb1a4af, store: [table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:14:51,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:51,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831351881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:51,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:51,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:51,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:51,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:51,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831351884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:51,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:51,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:51,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:51,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:51,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831351887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:51,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:51,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831351888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:51,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/8bbbc2dea6984a3893124f53ecb1a4af is 175, key is test_row_0/A:col10/1731831291846/Put/seqid=0 2024-11-17T08:14:51,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:51,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831351888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:51,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:51,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:51,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:51,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:51,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:51,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:51,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:51,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:51,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:51,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741957_1133 (size=39549) 2024-11-17T08:14:51,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T08:14:51,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T08:14:51,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:51,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. as already flushing 2024-11-17T08:14:51,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:51,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:51,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:51,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:52,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:52,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831352004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:52,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:52,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:52,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831352004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:52,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:52,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:52,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831352004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:52,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:52,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:52,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831352004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:52,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831352005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:52,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,008 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:52,008 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,008 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:52,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:52,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T08:14:52,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:52,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. as already flushing 2024-11-17T08:14:52,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:52,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:52,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:52,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:52,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T08:14:52,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831352213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:52,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831352214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:52,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831352214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831352214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:52,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:52,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831352223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:52,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:52,270 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T08:14:52,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T08:14:52,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:52,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. as already flushing 2024-11-17T08:14:52,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:52,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:52,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:52,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:52,313 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=64, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/8bbbc2dea6984a3893124f53ecb1a4af 2024-11-17T08:14:52,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/561d7b256cb44f28a300929a896c231a is 50, key is test_row_0/B:col10/1731831291846/Put/seqid=0 2024-11-17T08:14:52,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741958_1134 (size=12001) 2024-11-17T08:14:52,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/561d7b256cb44f28a300929a896c231a 2024-11-17T08:14:52,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/d656683944ea4588a954c236f733997f is 50, key is test_row_0/C:col10/1731831291846/Put/seqid=0 2024-11-17T08:14:52,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741959_1135 (size=12001) 2024-11-17T08:14:52,359 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/d656683944ea4588a954c236f733997f 2024-11-17T08:14:52,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/8bbbc2dea6984a3893124f53ecb1a4af as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/8bbbc2dea6984a3893124f53ecb1a4af 2024-11-17T08:14:52,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/8bbbc2dea6984a3893124f53ecb1a4af, entries=200, sequenceid=64, filesize=38.6 K 2024-11-17T08:14:52,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/561d7b256cb44f28a300929a896c231a as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/561d7b256cb44f28a300929a896c231a 2024-11-17T08:14:52,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/561d7b256cb44f28a300929a896c231a, entries=150, sequenceid=64, filesize=11.7 K 2024-11-17T08:14:52,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/d656683944ea4588a954c236f733997f as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/d656683944ea4588a954c236f733997f 2024-11-17T08:14:52,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/d656683944ea4588a954c236f733997f, entries=150, sequenceid=64, filesize=11.7 K 2024-11-17T08:14:52,395 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for e8e9eb0dab895bf5f9f5249707e9712d in 548ms, sequenceid=64, compaction requested=true 2024-11-17T08:14:52,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e8e9eb0dab895bf5f9f5249707e9712d: 2024-11-17T08:14:52,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8e9eb0dab895bf5f9f5249707e9712d:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:14:52,396 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:14:52,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:52,396 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:14:52,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8e9eb0dab895bf5f9f5249707e9712d:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:14:52,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:52,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8e9eb0dab895bf5f9f5249707e9712d:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:14:52,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:14:52,397 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:14:52,397 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): e8e9eb0dab895bf5f9f5249707e9712d/A is initiating minor compaction (all files) 2024-11-17T08:14:52,397 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8e9eb0dab895bf5f9f5249707e9712d/A in TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:52,397 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/d9356a37f801441e84c1577c0e921067, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/035678266982435f8d6ba4d2579d6ee2, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/8bbbc2dea6984a3893124f53ecb1a4af] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp, totalSize=99.1 K 2024-11-17T08:14:52,397 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:52,398 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. files: [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/d9356a37f801441e84c1577c0e921067, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/035678266982435f8d6ba4d2579d6ee2, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/8bbbc2dea6984a3893124f53ecb1a4af] 2024-11-17T08:14:52,399 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:14:52,399 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): e8e9eb0dab895bf5f9f5249707e9712d/B is initiating minor compaction (all files) 2024-11-17T08:14:52,399 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8e9eb0dab895bf5f9f5249707e9712d/B in TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:52,399 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/d33819c99464452cbdf3e97c74c65e94, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/52920993ae8c409b83231c9bab4f8da5, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/561d7b256cb44f28a300929a896c231a] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp, totalSize=35.2 K 2024-11-17T08:14:52,399 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9356a37f801441e84c1577c0e921067, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731831290690 2024-11-17T08:14:52,400 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting d33819c99464452cbdf3e97c74c65e94, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731831290690 2024-11-17T08:14:52,400 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 035678266982435f8d6ba4d2579d6ee2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731831290805 2024-11-17T08:14:52,400 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 52920993ae8c409b83231c9bab4f8da5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731831290805 2024-11-17T08:14:52,400 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bbbc2dea6984a3893124f53ecb1a4af, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731831291175 2024-11-17T08:14:52,401 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 561d7b256cb44f28a300929a896c231a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731831291215 2024-11-17T08:14:52,414 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8e9eb0dab895bf5f9f5249707e9712d#B#compaction#117 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:14:52,415 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/6f978e293b8340c7af15278aeb362224 is 50, key is test_row_0/B:col10/1731831291846/Put/seqid=0 2024-11-17T08:14:52,425 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:14:52,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T08:14:52,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:52,433 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing e8e9eb0dab895bf5f9f5249707e9712d 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-17T08:14:52,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=A 2024-11-17T08:14:52,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:52,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=B 2024-11-17T08:14:52,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:52,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=C 2024-11-17T08:14:52,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:52,439 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117703960096d394705b46c3c2f9555bb9f_e8e9eb0dab895bf5f9f5249707e9712d store=[table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:14:52,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T08:14:52,447 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117703960096d394705b46c3c2f9555bb9f_e8e9eb0dab895bf5f9f5249707e9712d, store=[table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:14:52,447 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117703960096d394705b46c3c2f9555bb9f_e8e9eb0dab895bf5f9f5249707e9712d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:14:52,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741960_1136 (size=12104) 2024-11-17T08:14:52,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741961_1137 (size=4469) 2024-11-17T08:14:52,476 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8e9eb0dab895bf5f9f5249707e9712d#A#compaction#118 average throughput is 0.48 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:14:52,479 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/af11922a4ebe4482b198ce37e26b2451 is 175, key is test_row_0/A:col10/1731831291846/Put/seqid=0 2024-11-17T08:14:52,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f8078879951d4a0b84993017e0b4572b_e8e9eb0dab895bf5f9f5249707e9712d is 50, key is test_row_0/A:col10/1731831291867/Put/seqid=0 2024-11-17T08:14:52,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741962_1138 (size=31058) 2024-11-17T08:14:52,505 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/af11922a4ebe4482b198ce37e26b2451 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/af11922a4ebe4482b198ce37e26b2451 2024-11-17T08:14:52,517 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8e9eb0dab895bf5f9f5249707e9712d/A of e8e9eb0dab895bf5f9f5249707e9712d into af11922a4ebe4482b198ce37e26b2451(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:14:52,517 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8e9eb0dab895bf5f9f5249707e9712d: 2024-11-17T08:14:52,517 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., storeName=e8e9eb0dab895bf5f9f5249707e9712d/A, priority=13, startTime=1731831292396; duration=0sec 2024-11-17T08:14:52,518 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:14:52,518 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8e9eb0dab895bf5f9f5249707e9712d:A 2024-11-17T08:14:52,518 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:14:52,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741963_1139 (size=12154) 2024-11-17T08:14:52,519 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:14:52,519 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): e8e9eb0dab895bf5f9f5249707e9712d/C is initiating minor compaction (all files) 2024-11-17T08:14:52,519 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8e9eb0dab895bf5f9f5249707e9712d/C in TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:52,519 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/bfae669b183f49cea2b6fb32e037b788, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/5580bc6eec3a41d497d77687f580acdc, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/d656683944ea4588a954c236f733997f] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp, totalSize=35.2 K 2024-11-17T08:14:52,520 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfae669b183f49cea2b6fb32e037b788, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731831290690 2024-11-17T08:14:52,520 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5580bc6eec3a41d497d77687f580acdc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731831290805 2024-11-17T08:14:52,520 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting d656683944ea4588a954c236f733997f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731831291215 2024-11-17T08:14:52,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. as already flushing 2024-11-17T08:14:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:52,538 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8e9eb0dab895bf5f9f5249707e9712d#C#compaction#120 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:14:52,538 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/fbe39c2a136f474ab01388d9c0ce7794 is 50, key is test_row_0/C:col10/1731831291846/Put/seqid=0 2024-11-17T08:14:52,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:52,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831352536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:52,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:52,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831352540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:52,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:52,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:52,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831352544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:52,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831352546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:52,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:52,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831352546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:52,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:52,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:52,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:52,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741964_1140 (size=12104) 2024-11-17T08:14:52,574 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/fbe39c2a136f474ab01388d9c0ce7794 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/fbe39c2a136f474ab01388d9c0ce7794 2024-11-17T08:14:52,583 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8e9eb0dab895bf5f9f5249707e9712d/C of e8e9eb0dab895bf5f9f5249707e9712d into fbe39c2a136f474ab01388d9c0ce7794(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:14:52,583 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8e9eb0dab895bf5f9f5249707e9712d: 2024-11-17T08:14:52,583 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., storeName=e8e9eb0dab895bf5f9f5249707e9712d/C, priority=13, startTime=1731831292396; duration=0sec 2024-11-17T08:14:52,583 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:52,583 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8e9eb0dab895bf5f9f5249707e9712d:C 2024-11-17T08:14:52,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:52,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831352654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:52,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:52,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831352654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:52,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:52,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:52,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:52,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831352663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:52,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:52,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831352663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:52,664 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,664 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:52,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831352664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:52,704 INFO [master/e3d0adb87c01:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-17T08:14:52,704 INFO [master/e3d0adb87c01:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-17T08:14:52,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:52,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831352864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:52,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:52,870 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/6f978e293b8340c7af15278aeb362224 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/6f978e293b8340c7af15278aeb362224 2024-11-17T08:14:52,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:52,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831352865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:52,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:52,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831352874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:52,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831352874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:52,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:52,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831352874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:52,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:52,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:52,877 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8e9eb0dab895bf5f9f5249707e9712d/B of e8e9eb0dab895bf5f9f5249707e9712d into 6f978e293b8340c7af15278aeb362224(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:14:52,877 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8e9eb0dab895bf5f9f5249707e9712d: 2024-11-17T08:14:52,877 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., storeName=e8e9eb0dab895bf5f9f5249707e9712d/B, priority=13, startTime=1731831292396; duration=0sec 2024-11-17T08:14:52,878 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:52,878 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8e9eb0dab895bf5f9f5249707e9712d:B 2024-11-17T08:14:52,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:14:52,933 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f8078879951d4a0b84993017e0b4572b_e8e9eb0dab895bf5f9f5249707e9712d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f8078879951d4a0b84993017e0b4572b_e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:52,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/9ded661498cc4563a13c4dc2538b2678, store: [table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:14:52,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/9ded661498cc4563a13c4dc2538b2678 is 175, key is test_row_0/A:col10/1731831291867/Put/seqid=0 2024-11-17T08:14:52,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T08:14:52,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741965_1141 (size=30955) 2024-11-17T08:14:52,962 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=83, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/9ded661498cc4563a13c4dc2538b2678 2024-11-17T08:14:52,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/4fb8f9c508174e65ad76e2dc8e5cb127 is 50, key is test_row_0/B:col10/1731831291867/Put/seqid=0 2024-11-17T08:14:52,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741966_1142 (size=12001) 2024-11-17T08:14:53,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:53,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831353173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:53,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:53,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:53,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:53,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:53,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831353184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:53,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:53,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831353184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:53,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:53,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:53,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:53,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:53,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:53,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:53,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:53,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831353184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:53,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:53,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:53,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:53,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:53,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831353184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:53,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:53,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:53,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:53,375 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/4fb8f9c508174e65ad76e2dc8e5cb127 2024-11-17T08:14:53,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/a447e26e13e548848a5ab4d1f53c782c is 50, key is test_row_0/C:col10/1731831291867/Put/seqid=0 2024-11-17T08:14:53,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741967_1143 (size=12001) 2024-11-17T08:14:53,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:53,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831353683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:53,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:53,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:53,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:53,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:53,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831353694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:53,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:53,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:53,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:53,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:53,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831353694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:53,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:53,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:53,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:53,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:53,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831353694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:53,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:53,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:53,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:53,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:53,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831353694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:53,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:53,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:53,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:53,789 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/a447e26e13e548848a5ab4d1f53c782c 2024-11-17T08:14:53,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/9ded661498cc4563a13c4dc2538b2678 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/9ded661498cc4563a13c4dc2538b2678 2024-11-17T08:14:53,799 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/9ded661498cc4563a13c4dc2538b2678, entries=150, sequenceid=83, filesize=30.2 K 2024-11-17T08:14:53,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/4fb8f9c508174e65ad76e2dc8e5cb127 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/4fb8f9c508174e65ad76e2dc8e5cb127 2024-11-17T08:14:53,807 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/4fb8f9c508174e65ad76e2dc8e5cb127, entries=150, sequenceid=83, filesize=11.7 K 2024-11-17T08:14:53,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/a447e26e13e548848a5ab4d1f53c782c as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/a447e26e13e548848a5ab4d1f53c782c 2024-11-17T08:14:53,815 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/a447e26e13e548848a5ab4d1f53c782c, entries=150, sequenceid=83, filesize=11.7 K 2024-11-17T08:14:53,816 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for e8e9eb0dab895bf5f9f5249707e9712d in 1383ms, sequenceid=83, compaction requested=false 2024-11-17T08:14:53,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for e8e9eb0dab895bf5f9f5249707e9712d: 2024-11-17T08:14:53,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:53,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-17T08:14:53,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-17T08:14:53,823 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-17T08:14:53,823 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0000 sec 2024-11-17T08:14:53,825 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.0090 sec 2024-11-17T08:14:53,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T08:14:53,964 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:14:53,965 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:14:53,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-17T08:14:53,968 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:14:53,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T08:14:53,969 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:14:53,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:14:54,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T08:14:54,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T08:14:54,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:54,126 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing e8e9eb0dab895bf5f9f5249707e9712d 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-17T08:14:54,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=A 2024-11-17T08:14:54,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:54,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=B 2024-11-17T08:14:54,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:54,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=C 2024-11-17T08:14:54,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:54,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117cb45eae08f1146e29e7d20f781afe997_e8e9eb0dab895bf5f9f5249707e9712d is 50, key is test_row_0/A:col10/1731831292534/Put/seqid=0 2024-11-17T08:14:54,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741968_1144 (size=12154) 2024-11-17T08:14:54,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:14:54,167 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117cb45eae08f1146e29e7d20f781afe997_e8e9eb0dab895bf5f9f5249707e9712d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117cb45eae08f1146e29e7d20f781afe997_e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:54,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/502e212023934fcbbb8db7e5a9b75e07, store: [table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:14:54,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/502e212023934fcbbb8db7e5a9b75e07 is 175, key is test_row_0/A:col10/1731831292534/Put/seqid=0 2024-11-17T08:14:54,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741969_1145 (size=30955) 2024-11-17T08:14:54,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T08:14:54,581 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=100, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/502e212023934fcbbb8db7e5a9b75e07 2024-11-17T08:14:54,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T08:14:54,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/a25b870fe4084e74a29cd43fb29283ba is 50, key is test_row_0/B:col10/1731831292534/Put/seqid=0 2024-11-17T08:14:54,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741970_1146 (size=12001) 2024-11-17T08:14:54,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. as already flushing 2024-11-17T08:14:54,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:54,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:54,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831354737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:54,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:54,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:54,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831354738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:54,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:54,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:54,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:54,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:54,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:54,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:54,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831354738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:54,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:54,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831354739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:54,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:54,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831354744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:54,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:54,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:54,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:54,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:54,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:54,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:54,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:54,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:54,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:54,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:54,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831354855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:54,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:54,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:54,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:54,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831354855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:54,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:54,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:54,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831354855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:54,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:54,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:54,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831354855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:54,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:54,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:54,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:54,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:54,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:54,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:54,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831354855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:54,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:54,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:54,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:55,012 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/a25b870fe4084e74a29cd43fb29283ba 2024-11-17T08:14:55,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/c45c99774ed44192a577873cc1c0f131 is 50, key is test_row_0/C:col10/1731831292534/Put/seqid=0 2024-11-17T08:14:55,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741971_1147 (size=12001) 2024-11-17T08:14:55,055 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/c45c99774ed44192a577873cc1c0f131 2024-11-17T08:14:55,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/502e212023934fcbbb8db7e5a9b75e07 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/502e212023934fcbbb8db7e5a9b75e07 2024-11-17T08:14:55,071 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/502e212023934fcbbb8db7e5a9b75e07, entries=150, sequenceid=100, filesize=30.2 K 2024-11-17T08:14:55,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/a25b870fe4084e74a29cd43fb29283ba as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/a25b870fe4084e74a29cd43fb29283ba 2024-11-17T08:14:55,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:55,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831355074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831355074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:55,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:55,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:55,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:55,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831355074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831355075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:55,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:55,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:55,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:55,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:55,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831355075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:55,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:55,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:55,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:55,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:55,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:55,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:55,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:55,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:55,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:55,081 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/a25b870fe4084e74a29cd43fb29283ba, entries=150, sequenceid=100, filesize=11.7 K 2024-11-17T08:14:55,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/c45c99774ed44192a577873cc1c0f131 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/c45c99774ed44192a577873cc1c0f131 2024-11-17T08:14:55,103 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/c45c99774ed44192a577873cc1c0f131, entries=150, sequenceid=100, filesize=11.7 K 2024-11-17T08:14:55,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T08:14:55,106 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for e8e9eb0dab895bf5f9f5249707e9712d in 980ms, sequenceid=100, compaction requested=true 2024-11-17T08:14:55,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for e8e9eb0dab895bf5f9f5249707e9712d: 2024-11-17T08:14:55,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:55,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-17T08:14:55,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-17T08:14:55,112 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-17T08:14:55,112 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1400 sec 2024-11-17T08:14:55,114 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.1480 sec 2024-11-17T08:14:55,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:55,388 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e8e9eb0dab895bf5f9f5249707e9712d 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-17T08:14:55,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=A 2024-11-17T08:14:55,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:55,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=B 2024-11-17T08:14:55,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:55,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=C 2024-11-17T08:14:55,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:55,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111756914b63d1df4d219479a3752a953ca4_e8e9eb0dab895bf5f9f5249707e9712d is 50, key is test_row_0/A:col10/1731831294734/Put/seqid=0 2024-11-17T08:14:55,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:55,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:55,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831355399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:55,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831355406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:55,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:55,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831355407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:55,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:55,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:55,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:55,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:55,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:55,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:55,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:55,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:55,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:55,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:55,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831355417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:55,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:55,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831355417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:55,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:55,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:55,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:55,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:55,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:55,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:55,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741972_1148 (size=12154) 2024-11-17T08:14:55,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:55,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831355523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:55,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:55,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:55,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:55,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:55,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831355523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:55,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:55,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831355523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:55,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:55,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:55,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:55,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:55,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:55,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:55,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831355534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:55,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:55,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:55,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:55,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:55,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831355534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:55,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:55,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:55,541 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:55,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:55,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831355734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:55,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:55,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:55,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831355734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:55,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831355734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:55,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:55,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:55,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:55,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:55,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:55,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:55,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:55,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:55,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:55,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:55,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831355743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:55,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:55,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:55,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:55,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831355743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:55,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:55,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:55,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:55,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:55,830 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:14:55,834 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111756914b63d1df4d219479a3752a953ca4_e8e9eb0dab895bf5f9f5249707e9712d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111756914b63d1df4d219479a3752a953ca4_e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:55,835 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/3fccda8059494b97a3b5fd25c8e73a10, store: [table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:14:55,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/3fccda8059494b97a3b5fd25c8e73a10 is 175, key is test_row_0/A:col10/1731831294734/Put/seqid=0 2024-11-17T08:14:55,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741973_1149 (size=30955) 2024-11-17T08:14:56,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:56,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831356044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:56,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:56,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:56,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831356044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:56,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:56,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:56,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831356045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:56,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831356054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:56,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:56,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:56,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:56,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831356054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:56,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:56,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:56,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:56,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:56,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T08:14:56,124 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:14:56,124 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:14:56,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-17T08:14:56,128 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:14:56,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T08:14:56,129 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:14:56,129 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:14:56,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T08:14:56,240 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/3fccda8059494b97a3b5fd25c8e73a10 2024-11-17T08:14:56,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/aa5dab037cb346db9495e56978eec622 is 50, key is test_row_0/B:col10/1731831294734/Put/seqid=0 2024-11-17T08:14:56,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741974_1150 (size=12001) 2024-11-17T08:14:56,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/aa5dab037cb346db9495e56978eec622 2024-11-17T08:14:56,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/2f57219a05104260afa417a50ecc5cf5 is 50, key is test_row_0/C:col10/1731831294734/Put/seqid=0 2024-11-17T08:14:56,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741975_1151 (size=12001) 2024-11-17T08:14:56,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T08:14:56,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:56,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. as already flushing 2024-11-17T08:14:56,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:56,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:56,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:56,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/2f57219a05104260afa417a50ecc5cf5 2024-11-17T08:14:56,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:56,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/3fccda8059494b97a3b5fd25c8e73a10 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/3fccda8059494b97a3b5fd25c8e73a10 2024-11-17T08:14:56,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/3fccda8059494b97a3b5fd25c8e73a10, entries=150, sequenceid=119, filesize=30.2 K 2024-11-17T08:14:56,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/aa5dab037cb346db9495e56978eec622 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/aa5dab037cb346db9495e56978eec622 2024-11-17T08:14:56,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/aa5dab037cb346db9495e56978eec622, entries=150, sequenceid=119, filesize=11.7 K 2024-11-17T08:14:56,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/2f57219a05104260afa417a50ecc5cf5 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/2f57219a05104260afa417a50ecc5cf5 2024-11-17T08:14:56,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/2f57219a05104260afa417a50ecc5cf5, entries=150, sequenceid=119, filesize=11.7 K 2024-11-17T08:14:56,326 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for e8e9eb0dab895bf5f9f5249707e9712d in 939ms, sequenceid=119, compaction requested=true 2024-11-17T08:14:56,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e8e9eb0dab895bf5f9f5249707e9712d: 2024-11-17T08:14:56,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8e9eb0dab895bf5f9f5249707e9712d:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:14:56,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:56,326 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T08:14:56,326 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T08:14:56,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8e9eb0dab895bf5f9f5249707e9712d:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:14:56,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:56,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8e9eb0dab895bf5f9f5249707e9712d:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:14:56,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:14:56,329 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 123923 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T08:14:56,329 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T08:14:56,329 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): e8e9eb0dab895bf5f9f5249707e9712d/A is initiating minor compaction (all files) 2024-11-17T08:14:56,329 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): e8e9eb0dab895bf5f9f5249707e9712d/B is initiating minor compaction (all files) 2024-11-17T08:14:56,329 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8e9eb0dab895bf5f9f5249707e9712d/B in TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:56,329 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8e9eb0dab895bf5f9f5249707e9712d/A in TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:56,329 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/af11922a4ebe4482b198ce37e26b2451, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/9ded661498cc4563a13c4dc2538b2678, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/502e212023934fcbbb8db7e5a9b75e07, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/3fccda8059494b97a3b5fd25c8e73a10] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp, totalSize=121.0 K 2024-11-17T08:14:56,329 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/6f978e293b8340c7af15278aeb362224, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/4fb8f9c508174e65ad76e2dc8e5cb127, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/a25b870fe4084e74a29cd43fb29283ba, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/aa5dab037cb346db9495e56978eec622] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp, totalSize=47.0 K 2024-11-17T08:14:56,329 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:56,329 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. files: [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/af11922a4ebe4482b198ce37e26b2451, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/9ded661498cc4563a13c4dc2538b2678, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/502e212023934fcbbb8db7e5a9b75e07, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/3fccda8059494b97a3b5fd25c8e73a10] 2024-11-17T08:14:56,330 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting af11922a4ebe4482b198ce37e26b2451, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731831291215 2024-11-17T08:14:56,330 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f978e293b8340c7af15278aeb362224, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731831291215 2024-11-17T08:14:56,330 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ded661498cc4563a13c4dc2538b2678, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731831291867 2024-11-17T08:14:56,330 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fb8f9c508174e65ad76e2dc8e5cb127, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731831291867 2024-11-17T08:14:56,331 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 502e212023934fcbbb8db7e5a9b75e07, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731831292534 2024-11-17T08:14:56,331 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting a25b870fe4084e74a29cd43fb29283ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731831292534 2024-11-17T08:14:56,331 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fccda8059494b97a3b5fd25c8e73a10, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731831294734 2024-11-17T08:14:56,332 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting aa5dab037cb346db9495e56978eec622, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731831294734 2024-11-17T08:14:56,351 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8e9eb0dab895bf5f9f5249707e9712d#B#compaction#129 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:14:56,352 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/1ba51ab6777f4c9b830035819380b36a is 50, key is test_row_0/B:col10/1731831294734/Put/seqid=0 2024-11-17T08:14:56,354 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:14:56,361 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111725850414197f49ecac52a1640d6e9239_e8e9eb0dab895bf5f9f5249707e9712d store=[table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:14:56,365 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111725850414197f49ecac52a1640d6e9239_e8e9eb0dab895bf5f9f5249707e9712d, store=[table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:14:56,365 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111725850414197f49ecac52a1640d6e9239_e8e9eb0dab895bf5f9f5249707e9712d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:14:56,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741976_1152 (size=12241) 2024-11-17T08:14:56,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741977_1153 (size=4469) 2024-11-17T08:14:56,404 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8e9eb0dab895bf5f9f5249707e9712d#A#compaction#130 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:14:56,404 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/8859163ffe6f42bb98ff04bf2dbad874 is 175, key is test_row_0/A:col10/1731831294734/Put/seqid=0 2024-11-17T08:14:56,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741978_1154 (size=31195) 2024-11-17T08:14:56,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T08:14:56,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:56,436 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing e8e9eb0dab895bf5f9f5249707e9712d 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-17T08:14:56,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=A 2024-11-17T08:14:56,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:56,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=B 2024-11-17T08:14:56,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:56,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=C 2024-11-17T08:14:56,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:56,438 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/8859163ffe6f42bb98ff04bf2dbad874 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/8859163ffe6f42bb98ff04bf2dbad874 2024-11-17T08:14:56,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T08:14:56,452 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e8e9eb0dab895bf5f9f5249707e9712d/A of e8e9eb0dab895bf5f9f5249707e9712d into 8859163ffe6f42bb98ff04bf2dbad874(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:14:56,452 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8e9eb0dab895bf5f9f5249707e9712d: 2024-11-17T08:14:56,452 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., storeName=e8e9eb0dab895bf5f9f5249707e9712d/A, priority=12, startTime=1731831296326; duration=0sec 2024-11-17T08:14:56,452 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:14:56,452 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8e9eb0dab895bf5f9f5249707e9712d:A 2024-11-17T08:14:56,452 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T08:14:56,454 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T08:14:56,454 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): e8e9eb0dab895bf5f9f5249707e9712d/C is initiating minor compaction (all files) 2024-11-17T08:14:56,454 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8e9eb0dab895bf5f9f5249707e9712d/C in TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:56,455 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/fbe39c2a136f474ab01388d9c0ce7794, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/a447e26e13e548848a5ab4d1f53c782c, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/c45c99774ed44192a577873cc1c0f131, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/2f57219a05104260afa417a50ecc5cf5] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp, totalSize=47.0 K 2024-11-17T08:14:56,456 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbe39c2a136f474ab01388d9c0ce7794, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731831291215 2024-11-17T08:14:56,456 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting a447e26e13e548848a5ab4d1f53c782c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731831291867 2024-11-17T08:14:56,457 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting c45c99774ed44192a577873cc1c0f131, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731831292534 2024-11-17T08:14:56,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b9d628d21e41486ea93df90ee59e6a8d_e8e9eb0dab895bf5f9f5249707e9712d is 50, key is test_row_0/A:col10/1731831295395/Put/seqid=0 2024-11-17T08:14:56,461 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f57219a05104260afa417a50ecc5cf5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731831294734 2024-11-17T08:14:56,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741979_1155 (size=12254) 2024-11-17T08:14:56,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:14:56,496 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b9d628d21e41486ea93df90ee59e6a8d_e8e9eb0dab895bf5f9f5249707e9712d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b9d628d21e41486ea93df90ee59e6a8d_e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:56,498 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8e9eb0dab895bf5f9f5249707e9712d#C#compaction#132 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:14:56,498 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/4ae8f1a609154636861125c4c5b4aeec is 50, key is test_row_0/C:col10/1731831294734/Put/seqid=0 2024-11-17T08:14:56,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/3f7fbd1842fb4fcd83b6ad5fea8c02b1, store: [table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:14:56,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/3f7fbd1842fb4fcd83b6ad5fea8c02b1 is 175, key is test_row_0/A:col10/1731831295395/Put/seqid=0 2024-11-17T08:14:56,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741980_1156 (size=31055) 2024-11-17T08:14:56,523 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/3f7fbd1842fb4fcd83b6ad5fea8c02b1 2024-11-17T08:14:56,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/b7e9b9ee9a374c4bae78f3a145f0966e is 50, key is test_row_0/B:col10/1731831295395/Put/seqid=0 2024-11-17T08:14:56,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741981_1157 (size=12241) 2024-11-17T08:14:56,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. as already flushing 2024-11-17T08:14:56,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:56,556 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/4ae8f1a609154636861125c4c5b4aeec as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/4ae8f1a609154636861125c4c5b4aeec 2024-11-17T08:14:56,566 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e8e9eb0dab895bf5f9f5249707e9712d/C of e8e9eb0dab895bf5f9f5249707e9712d into 4ae8f1a609154636861125c4c5b4aeec(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:14:56,566 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8e9eb0dab895bf5f9f5249707e9712d: 2024-11-17T08:14:56,566 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., storeName=e8e9eb0dab895bf5f9f5249707e9712d/C, priority=12, startTime=1731831296327; duration=0sec 2024-11-17T08:14:56,566 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:56,566 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8e9eb0dab895bf5f9f5249707e9712d:C 2024-11-17T08:14:56,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741982_1158 (size=12101) 2024-11-17T08:14:56,578 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/b7e9b9ee9a374c4bae78f3a145f0966e 2024-11-17T08:14:56,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:56,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831356574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:56,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:56,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:56,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:56,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:56,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831356580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:56,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:56,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:56,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:56,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/18079be5841a4a6bae392310f6b7e0c7 is 50, key is test_row_0/C:col10/1731831295395/Put/seqid=0 2024-11-17T08:14:56,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:56,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831356582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:56,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:56,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:56,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831356585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:56,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:56,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:56,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:56,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:56,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:56,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:56,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831356586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:56,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:56,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:56,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:56,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741983_1159 (size=12101) 2024-11-17T08:14:56,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:56,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831356693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:56,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:56,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:56,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:56,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:56,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831356693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:56,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:56,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:56,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:56,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:56,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831356693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:56,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:56,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:56,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:56,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:56,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831356693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:56,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:56,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:56,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:56,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:56,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831356694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:56,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:56,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:56,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:56,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T08:14:56,793 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/1ba51ab6777f4c9b830035819380b36a as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/1ba51ab6777f4c9b830035819380b36a 2024-11-17T08:14:56,798 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e8e9eb0dab895bf5f9f5249707e9712d/B of e8e9eb0dab895bf5f9f5249707e9712d into 1ba51ab6777f4c9b830035819380b36a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:14:56,798 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8e9eb0dab895bf5f9f5249707e9712d: 2024-11-17T08:14:56,798 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., storeName=e8e9eb0dab895bf5f9f5249707e9712d/B, priority=12, startTime=1731831296326; duration=0sec 2024-11-17T08:14:56,798 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:56,798 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8e9eb0dab895bf5f9f5249707e9712d:B 2024-11-17T08:14:56,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:56,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831356903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:56,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:56,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:56,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831356903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:56,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:56,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:56,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:56,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:56,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:56,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831356903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:56,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:56,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:56,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:56,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:56,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:56,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831356904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:56,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:56,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:56,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:56,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:56,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831356904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:56,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:56,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:56,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:56,997 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/18079be5841a4a6bae392310f6b7e0c7 2024-11-17T08:14:57,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/3f7fbd1842fb4fcd83b6ad5fea8c02b1 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/3f7fbd1842fb4fcd83b6ad5fea8c02b1 2024-11-17T08:14:57,014 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/3f7fbd1842fb4fcd83b6ad5fea8c02b1, entries=150, sequenceid=133, filesize=30.3 K 2024-11-17T08:14:57,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/b7e9b9ee9a374c4bae78f3a145f0966e as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/b7e9b9ee9a374c4bae78f3a145f0966e 2024-11-17T08:14:57,026 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/b7e9b9ee9a374c4bae78f3a145f0966e, entries=150, sequenceid=133, filesize=11.8 K 2024-11-17T08:14:57,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/18079be5841a4a6bae392310f6b7e0c7 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/18079be5841a4a6bae392310f6b7e0c7 2024-11-17T08:14:57,037 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/18079be5841a4a6bae392310f6b7e0c7, entries=150, sequenceid=133, filesize=11.8 K 2024-11-17T08:14:57,038 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for e8e9eb0dab895bf5f9f5249707e9712d in 602ms, sequenceid=133, compaction requested=false 2024-11-17T08:14:57,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for e8e9eb0dab895bf5f9f5249707e9712d: 2024-11-17T08:14:57,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:57,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-17T08:14:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-17T08:14:57,041 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-17T08:14:57,041 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 910 msec 2024-11-17T08:14:57,043 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 918 msec 2024-11-17T08:14:57,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:57,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e8e9eb0dab895bf5f9f5249707e9712d 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-17T08:14:57,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=A 2024-11-17T08:14:57,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:57,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=B 2024-11-17T08:14:57,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:57,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=C 2024-11-17T08:14:57,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:57,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111780ba616916f84e0bab8918828709beca_e8e9eb0dab895bf5f9f5249707e9712d is 50, key is test_row_0/A:col10/1731831296571/Put/seqid=0 2024-11-17T08:14:57,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831357226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:57,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:57,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831357228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:57,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:57,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:57,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831357229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:57,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:57,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831357230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:57,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:57,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:57,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:57,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831357233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:57,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:57,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:57,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:57,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:57,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:57,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:57,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741984_1160 (size=17284) 2024-11-17T08:14:57,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T08:14:57,264 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:14:57,264 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:14:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-17T08:14:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T08:14:57,268 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:14:57,269 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:14:57,269 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:14:57,327 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-17T08:14:57,328 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-17T08:14:57,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831357343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:57,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:57,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:57,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:57,345 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831357344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:57,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:57,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:57,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:57,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831357344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:57,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:57,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:57,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:57,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831357344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:57,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831357344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:57,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:57,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:57,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:57,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:57,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T08:14:57,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T08:14:57,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:57,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. as already flushing 2024-11-17T08:14:57,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:57,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:57,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:57,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:57,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831357553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:57,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:57,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831357553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:57,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:57,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:57,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:57,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:57,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:57,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:57,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:57,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831357553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:57,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:57,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:57,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831357554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:57,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:57,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:57,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:57,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:57,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:57,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:57,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831357564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:57,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:57,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:57,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:57,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T08:14:57,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:57,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. as already flushing 2024-11-17T08:14:57,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:57,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:57,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:57,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:57,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T08:14:57,653 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:14:57,657 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111780ba616916f84e0bab8918828709beca_e8e9eb0dab895bf5f9f5249707e9712d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111780ba616916f84e0bab8918828709beca_e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:57,658 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/cd3807996e1e459a83b82c9455b07620, store: [table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:14:57,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/cd3807996e1e459a83b82c9455b07620 is 175, key is test_row_0/A:col10/1731831296571/Put/seqid=0 2024-11-17T08:14:57,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741985_1161 (size=48389) 2024-11-17T08:14:57,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T08:14:57,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:57,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. as already flushing 2024-11-17T08:14:57,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:57,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:57,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:57,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:57,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:57,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831357863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:57,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:57,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:57,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831357863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:57,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:57,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:57,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:57,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:57,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:57,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831357864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:57,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:57,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:57,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:57,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:57,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831357864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:57,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:57,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:57,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:57,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:57,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:57,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831357874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:57,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:57,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:57,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:57,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T08:14:57,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:57,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. as already flushing 2024-11-17T08:14:57,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:57,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:57,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:57,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:57,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T08:14:58,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T08:14:58,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:58,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. as already flushing 2024-11-17T08:14:58,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:58,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:58,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:58,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:58,064 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/cd3807996e1e459a83b82c9455b07620 2024-11-17T08:14:58,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/65bf1b6f20854dad8d26c3e460b97386 is 50, key is test_row_0/B:col10/1731831296571/Put/seqid=0 2024-11-17T08:14:58,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741986_1162 (size=12151) 2024-11-17T08:14:58,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T08:14:58,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:58,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. as already flushing 2024-11-17T08:14:58,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:58,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:58,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:58,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:58,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T08:14:58,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:58,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. as already flushing 2024-11-17T08:14:58,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:58,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:58,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:58,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:58,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:58,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831358374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:58,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:58,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:58,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831358374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:58,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:58,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:58,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:58,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:58,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:58,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831358374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:58,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:58,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:58,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:58,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831358374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:58,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:58,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:58,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:58,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:58,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:58,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:58,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831358383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:58,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:58,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:58,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:58,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T08:14:58,476 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/65bf1b6f20854dad8d26c3e460b97386 2024-11-17T08:14:58,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/a48393c568624f6dbf3a2c06e669c551 is 50, key is test_row_0/C:col10/1731831296571/Put/seqid=0 2024-11-17T08:14:58,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741987_1163 (size=12151) 2024-11-17T08:14:58,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T08:14:58,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:58,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. as already flushing 2024-11-17T08:14:58,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:58,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:58,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:58,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:58,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T08:14:58,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:58,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. as already flushing 2024-11-17T08:14:58,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:58,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:58,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:58,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:58,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T08:14:58,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:58,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. as already flushing 2024-11-17T08:14:58,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:58,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:58,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:58,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:14:58,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/a48393c568624f6dbf3a2c06e669c551 2024-11-17T08:14:58,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/cd3807996e1e459a83b82c9455b07620 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/cd3807996e1e459a83b82c9455b07620 2024-11-17T08:14:58,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/cd3807996e1e459a83b82c9455b07620, entries=250, sequenceid=156, filesize=47.3 K 2024-11-17T08:14:58,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/65bf1b6f20854dad8d26c3e460b97386 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/65bf1b6f20854dad8d26c3e460b97386 2024-11-17T08:14:58,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/65bf1b6f20854dad8d26c3e460b97386, entries=150, sequenceid=156, filesize=11.9 K 2024-11-17T08:14:58,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/a48393c568624f6dbf3a2c06e669c551 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/a48393c568624f6dbf3a2c06e669c551 2024-11-17T08:14:58,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/a48393c568624f6dbf3a2c06e669c551, entries=150, sequenceid=156, filesize=11.9 K 2024-11-17T08:14:58,911 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for e8e9eb0dab895bf5f9f5249707e9712d in 1693ms, sequenceid=156, compaction requested=true 2024-11-17T08:14:58,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e8e9eb0dab895bf5f9f5249707e9712d: 2024-11-17T08:14:58,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8e9eb0dab895bf5f9f5249707e9712d:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:14:58,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:58,912 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:14:58,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8e9eb0dab895bf5f9f5249707e9712d:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:14:58,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:58,912 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:14:58,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8e9eb0dab895bf5f9f5249707e9712d:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:14:58,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:14:58,913 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:14:58,913 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110639 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:14:58,913 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): e8e9eb0dab895bf5f9f5249707e9712d/B is initiating minor compaction (all files) 2024-11-17T08:14:58,913 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): e8e9eb0dab895bf5f9f5249707e9712d/A is initiating minor compaction (all files) 2024-11-17T08:14:58,913 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8e9eb0dab895bf5f9f5249707e9712d/B in TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:58,913 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8e9eb0dab895bf5f9f5249707e9712d/A in TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:58,913 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/8859163ffe6f42bb98ff04bf2dbad874, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/3f7fbd1842fb4fcd83b6ad5fea8c02b1, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/cd3807996e1e459a83b82c9455b07620] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp, totalSize=108.0 K 2024-11-17T08:14:58,913 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/1ba51ab6777f4c9b830035819380b36a, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/b7e9b9ee9a374c4bae78f3a145f0966e, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/65bf1b6f20854dad8d26c3e460b97386] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp, totalSize=35.6 K 2024-11-17T08:14:58,913 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:58,914 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. files: [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/8859163ffe6f42bb98ff04bf2dbad874, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/3f7fbd1842fb4fcd83b6ad5fea8c02b1, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/cd3807996e1e459a83b82c9455b07620] 2024-11-17T08:14:58,914 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8859163ffe6f42bb98ff04bf2dbad874, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731831294734 2024-11-17T08:14:58,914 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ba51ab6777f4c9b830035819380b36a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731831294734 2024-11-17T08:14:58,915 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting b7e9b9ee9a374c4bae78f3a145f0966e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731831295395 2024-11-17T08:14:58,915 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f7fbd1842fb4fcd83b6ad5fea8c02b1, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731831295395 2024-11-17T08:14:58,915 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 65bf1b6f20854dad8d26c3e460b97386, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731831296571 2024-11-17T08:14:58,915 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd3807996e1e459a83b82c9455b07620, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731831296571 2024-11-17T08:14:58,921 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:14:58,923 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8e9eb0dab895bf5f9f5249707e9712d#B#compaction#138 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:14:58,923 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/df59cc85272a48318542421eb21c0b0d is 50, key is test_row_0/B:col10/1731831296571/Put/seqid=0 2024-11-17T08:14:58,923 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117255dc71a93dd4dfab5448da904eeb42d_e8e9eb0dab895bf5f9f5249707e9712d store=[table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:14:58,926 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117255dc71a93dd4dfab5448da904eeb42d_e8e9eb0dab895bf5f9f5249707e9712d, store=[table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:14:58,927 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117255dc71a93dd4dfab5448da904eeb42d_e8e9eb0dab895bf5f9f5249707e9712d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:14:58,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741989_1165 (size=4469) 2024-11-17T08:14:58,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741988_1164 (size=12493) 2024-11-17T08:14:58,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T08:14:58,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:58,953 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing e8e9eb0dab895bf5f9f5249707e9712d 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-17T08:14:58,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=A 2024-11-17T08:14:58,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:58,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=B 2024-11-17T08:14:58,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:58,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=C 2024-11-17T08:14:58,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:14:58,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174491ff7f4e9d41c3838429493c156652_e8e9eb0dab895bf5f9f5249707e9712d is 50, key is test_row_0/A:col10/1731831297229/Put/seqid=0 2024-11-17T08:14:58,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741990_1166 (size=12304) 2024-11-17T08:14:58,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:14:58,974 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174491ff7f4e9d41c3838429493c156652_e8e9eb0dab895bf5f9f5249707e9712d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174491ff7f4e9d41c3838429493c156652_e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:58,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/652503f7b0864a8796c5c4fac9f9ca04, store: [table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:14:58,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/652503f7b0864a8796c5c4fac9f9ca04 is 175, key is test_row_0/A:col10/1731831297229/Put/seqid=0 2024-11-17T08:14:58,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741991_1167 (size=31105) 2024-11-17T08:14:59,340 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8e9eb0dab895bf5f9f5249707e9712d#A#compaction#139 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:14:59,341 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/0b33139bad554f7fa4442dd018ed2b9a is 175, key is test_row_0/A:col10/1731831296571/Put/seqid=0 2024-11-17T08:14:59,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741992_1168 (size=31447) 2024-11-17T08:14:59,357 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/df59cc85272a48318542421eb21c0b0d as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/df59cc85272a48318542421eb21c0b0d 2024-11-17T08:14:59,361 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8e9eb0dab895bf5f9f5249707e9712d/B of e8e9eb0dab895bf5f9f5249707e9712d into df59cc85272a48318542421eb21c0b0d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:14:59,361 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8e9eb0dab895bf5f9f5249707e9712d: 2024-11-17T08:14:59,361 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., storeName=e8e9eb0dab895bf5f9f5249707e9712d/B, priority=13, startTime=1731831298912; duration=0sec 2024-11-17T08:14:59,362 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:14:59,362 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8e9eb0dab895bf5f9f5249707e9712d:B 2024-11-17T08:14:59,362 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:14:59,363 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:14:59,363 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): e8e9eb0dab895bf5f9f5249707e9712d/C is initiating minor compaction (all files) 2024-11-17T08:14:59,363 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8e9eb0dab895bf5f9f5249707e9712d/C in TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:14:59,363 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/4ae8f1a609154636861125c4c5b4aeec, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/18079be5841a4a6bae392310f6b7e0c7, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/a48393c568624f6dbf3a2c06e669c551] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp, totalSize=35.6 K 2024-11-17T08:14:59,364 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ae8f1a609154636861125c4c5b4aeec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731831294734 2024-11-17T08:14:59,365 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 18079be5841a4a6bae392310f6b7e0c7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731831295395 2024-11-17T08:14:59,365 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting a48393c568624f6dbf3a2c06e669c551, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731831296571 2024-11-17T08:14:59,371 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8e9eb0dab895bf5f9f5249707e9712d#C#compaction#141 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:14:59,371 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/d964cdbe661d4f309880c19e824e4633 is 50, key is test_row_0/C:col10/1731831296571/Put/seqid=0 2024-11-17T08:14:59,382 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/652503f7b0864a8796c5c4fac9f9ca04 2024-11-17T08:14:59,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741993_1169 (size=12493) 2024-11-17T08:14:59,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:14:59,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. as already flushing 2024-11-17T08:14:59,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/51e7a59911804f44b6f252f29ef4a9df is 50, key is test_row_0/B:col10/1731831297229/Put/seqid=0 2024-11-17T08:14:59,399 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/d964cdbe661d4f309880c19e824e4633 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/d964cdbe661d4f309880c19e824e4633 2024-11-17T08:14:59,405 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8e9eb0dab895bf5f9f5249707e9712d/C of e8e9eb0dab895bf5f9f5249707e9712d into d964cdbe661d4f309880c19e824e4633(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:14:59,405 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8e9eb0dab895bf5f9f5249707e9712d: 2024-11-17T08:14:59,405 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., storeName=e8e9eb0dab895bf5f9f5249707e9712d/C, priority=13, startTime=1731831298912; duration=0sec 2024-11-17T08:14:59,405 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:59,405 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8e9eb0dab895bf5f9f5249707e9712d:C 2024-11-17T08:14:59,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741994_1170 (size=12151) 2024-11-17T08:14:59,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T08:14:59,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:59,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831359415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:59,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:59,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831359417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:59,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:59,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:59,425 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:59,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:59,425 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:59,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:59,425 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:59,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831359419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:59,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:59,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:59,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:59,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:59,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831359422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:59,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:59,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:59,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:59,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:59,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831359425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:59,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:59,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:59,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:59,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:59,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831359534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:59,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:59,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:59,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831359534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:59,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:59,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:59,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:59,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:59,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:59,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:59,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831359534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:59,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:59,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:59,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:59,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831359534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:59,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:59,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:59,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:59,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:59,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:59,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831359534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:59,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:59,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:59,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:59,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:59,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831359744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:59,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:59,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831359744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:59,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:59,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:59,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:59,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:59,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831359744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:59,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:59,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:59,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:59,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831359744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:59,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:59,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:14:59,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:59,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831359744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:14:59,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:59,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:59,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:59,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:59,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:59,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:59,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:14:59,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:14:59,750 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/0b33139bad554f7fa4442dd018ed2b9a as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/0b33139bad554f7fa4442dd018ed2b9a 2024-11-17T08:14:59,761 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8e9eb0dab895bf5f9f5249707e9712d/A of e8e9eb0dab895bf5f9f5249707e9712d into 0b33139bad554f7fa4442dd018ed2b9a(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:14:59,761 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8e9eb0dab895bf5f9f5249707e9712d: 2024-11-17T08:14:59,761 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., storeName=e8e9eb0dab895bf5f9f5249707e9712d/A, priority=13, startTime=1731831298912; duration=0sec 2024-11-17T08:14:59,761 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:14:59,761 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8e9eb0dab895bf5f9f5249707e9712d:A 2024-11-17T08:14:59,808 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/51e7a59911804f44b6f252f29ef4a9df 2024-11-17T08:14:59,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/3d738c8631314f489057c5dbe1f15c49 is 50, key is test_row_0/C:col10/1731831297229/Put/seqid=0 2024-11-17T08:14:59,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741995_1171 (size=12151) 2024-11-17T08:15:00,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:00,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831360054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:00,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:00,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:00,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831360054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:00,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:00,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:00,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:00,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:00,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:00,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831360054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:00,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:00,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:00,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:00,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831360054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:00,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:00,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:00,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:00,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:00,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831360054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:00,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:00,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:00,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:00,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:00,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:00,238 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/3d738c8631314f489057c5dbe1f15c49 2024-11-17T08:15:00,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/652503f7b0864a8796c5c4fac9f9ca04 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/652503f7b0864a8796c5c4fac9f9ca04 2024-11-17T08:15:00,250 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/652503f7b0864a8796c5c4fac9f9ca04, entries=150, sequenceid=168, filesize=30.4 K 2024-11-17T08:15:00,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/51e7a59911804f44b6f252f29ef4a9df as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/51e7a59911804f44b6f252f29ef4a9df 2024-11-17T08:15:00,255 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/51e7a59911804f44b6f252f29ef4a9df, entries=150, sequenceid=168, filesize=11.9 K 2024-11-17T08:15:00,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/3d738c8631314f489057c5dbe1f15c49 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/3d738c8631314f489057c5dbe1f15c49 2024-11-17T08:15:00,261 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/3d738c8631314f489057c5dbe1f15c49, entries=150, sequenceid=168, filesize=11.9 K 2024-11-17T08:15:00,262 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for e8e9eb0dab895bf5f9f5249707e9712d in 1309ms, sequenceid=168, compaction requested=false 2024-11-17T08:15:00,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for e8e9eb0dab895bf5f9f5249707e9712d: 2024-11-17T08:15:00,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:00,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-17T08:15:00,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-17T08:15:00,265 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-17T08:15:00,265 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9940 sec 2024-11-17T08:15:00,266 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 3.0000 sec 2024-11-17T08:15:00,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731831299417, max timestamp=1731831300565Num uniques 859; 2024-11-17T08:15:00,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T08:15:00,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731831299417, max timestamp=1731831300565Num uniques 859; 2024-11-17T08:15:00,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T08:15:00,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731831299417, max timestamp=1731831300565Num uniques 859; 2024-11-17T08:15:00,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T08:15:00,569 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e8e9eb0dab895bf5f9f5249707e9712d 3/3 column families, dataSize=26.84 KB heapSize=66.23 KB 2024-11-17T08:15:00,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=A 2024-11-17T08:15:00,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:15:00,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=B 2024-11-17T08:15:00,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:15:00,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=C 2024-11-17T08:15:00,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:15:00,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:15:00,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731831300576, max timestamp=1731831300595Num uniques 859; 2024-11-17T08:15:00,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731831300576, max timestamp=1731831300595Num uniques 859; 2024-11-17T08:15:00,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731831300576, max timestamp=1731831300595Num uniques 859; 2024-11-17T08:15:00,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-17T08:15:00,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-17T08:15:00,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-17T08:15:00,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:00,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831360616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:00,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:00,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:00,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:00,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:00,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831360616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:00,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:00,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831360617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:00,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:00,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:00,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:00,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:00,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831360618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:00,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:00,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831360620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:00,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:00,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:00,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:00,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:00,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:00,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:00,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:00,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:00,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:00,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e030e7a410064a1da01cdcbdb910a85b_e8e9eb0dab895bf5f9f5249707e9712d is 50, key is test_row_0/A:col10/1731831299417/Put/seqid=0 2024-11-17T08:15:00,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741996_1172 (size=12304) 2024-11-17T08:15:00,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:00,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831360734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:00,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:00,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:00,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831360734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:00,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:00,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:00,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:00,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:00,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:00,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:00,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:00,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831360734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:00,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831360734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:00,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:00,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:00,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:00,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:00,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:00,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:00,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831360734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:00,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:00,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:00,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:00,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:00,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:00,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831360944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:00,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:00,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:00,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831360944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:00,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:00,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:00,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:00,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:00,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831360944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:00,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:00,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:00,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:00,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:00,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:00,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:00,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831360944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:00,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:00,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:00,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:00,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:00,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831360954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:00,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:00,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:00,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:01,054 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:01,057 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e030e7a410064a1da01cdcbdb910a85b_e8e9eb0dab895bf5f9f5249707e9712d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e030e7a410064a1da01cdcbdb910a85b_e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:15:01,058 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/508cedb3d41d487d86365829e9eaf9a6, store: [table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:15:01,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/508cedb3d41d487d86365829e9eaf9a6 is 175, key is test_row_0/A:col10/1731831299417/Put/seqid=0 2024-11-17T08:15:01,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741997_1173 (size=31105) 2024-11-17T08:15:01,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:01,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831361253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:01,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:01,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:01,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:01,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831361253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:01,254 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:01,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:01,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:01,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:01,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:01,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831361264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:01,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:01,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831361264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:01,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:01,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:01,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:01,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:01,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:01,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:01,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:01,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831361264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:01,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:01,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:01,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:01,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T08:15:01,434 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:15:01,434 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:15:01,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-17T08:15:01,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T08:15:01,436 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:15:01,436 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:15:01,436 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:15:01,463 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=193, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/508cedb3d41d487d86365829e9eaf9a6 2024-11-17T08:15:01,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/525863c91fea452587a57482c149f887 is 50, key is test_row_0/B:col10/1731831299417/Put/seqid=0 2024-11-17T08:15:01,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741998_1174 (size=12151) 2024-11-17T08:15:01,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/525863c91fea452587a57482c149f887 2024-11-17T08:15:01,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/d65fb2fd06eb44ebb7aefcaacf4d051a is 50, key is test_row_0/C:col10/1731831299417/Put/seqid=0 2024-11-17T08:15:01,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741999_1175 (size=12151) 2024-11-17T08:15:01,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T08:15:01,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T08:15:01,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:01,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. as already flushing 2024-11-17T08:15:01,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:01,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:01,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:01,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:01,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T08:15:01,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:01,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. as already flushing 2024-11-17T08:15:01,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:01,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:01,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:01,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:01,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T08:15:01,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:01,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831361764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:01,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:01,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:01,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:01,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:01,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831361764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:01,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:01,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:01,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:01,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:01,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:01,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831361773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:01,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831361773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:01,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:01,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:01,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:01,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:01,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:01,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831361773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:01,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:01,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:01,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:01,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:01,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:01,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T08:15:01,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:01,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. as already flushing 2024-11-17T08:15:01,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:01,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:01,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:01,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:01,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/d65fb2fd06eb44ebb7aefcaacf4d051a 2024-11-17T08:15:01,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/508cedb3d41d487d86365829e9eaf9a6 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/508cedb3d41d487d86365829e9eaf9a6 2024-11-17T08:15:01,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/508cedb3d41d487d86365829e9eaf9a6, entries=150, sequenceid=193, filesize=30.4 K 2024-11-17T08:15:01,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/525863c91fea452587a57482c149f887 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/525863c91fea452587a57482c149f887 2024-11-17T08:15:01,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/525863c91fea452587a57482c149f887, entries=150, sequenceid=193, filesize=11.9 K 2024-11-17T08:15:01,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/d65fb2fd06eb44ebb7aefcaacf4d051a as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/d65fb2fd06eb44ebb7aefcaacf4d051a 2024-11-17T08:15:01,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/d65fb2fd06eb44ebb7aefcaacf4d051a, entries=150, sequenceid=193, filesize=11.9 K 2024-11-17T08:15:01,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=154.31 KB/158010 for e8e9eb0dab895bf5f9f5249707e9712d in 1357ms, sequenceid=193, compaction requested=true 2024-11-17T08:15:01,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e8e9eb0dab895bf5f9f5249707e9712d: 2024-11-17T08:15:01,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8e9eb0dab895bf5f9f5249707e9712d:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:15:01,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:01,926 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:15:01,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8e9eb0dab895bf5f9f5249707e9712d:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:15:01,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:01,926 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:15:01,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8e9eb0dab895bf5f9f5249707e9712d:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:15:01,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:15:01,927 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:15:01,927 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:15:01,927 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): e8e9eb0dab895bf5f9f5249707e9712d/A is initiating minor compaction (all files) 2024-11-17T08:15:01,927 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): e8e9eb0dab895bf5f9f5249707e9712d/B is initiating minor compaction (all files) 2024-11-17T08:15:01,927 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8e9eb0dab895bf5f9f5249707e9712d/A in TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:01,927 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8e9eb0dab895bf5f9f5249707e9712d/B in TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:01,927 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/0b33139bad554f7fa4442dd018ed2b9a, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/652503f7b0864a8796c5c4fac9f9ca04, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/508cedb3d41d487d86365829e9eaf9a6] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp, totalSize=91.5 K 2024-11-17T08:15:01,927 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/df59cc85272a48318542421eb21c0b0d, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/51e7a59911804f44b6f252f29ef4a9df, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/525863c91fea452587a57482c149f887] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp, totalSize=35.9 K 2024-11-17T08:15:01,927 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:01,927 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. files: [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/0b33139bad554f7fa4442dd018ed2b9a, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/652503f7b0864a8796c5c4fac9f9ca04, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/508cedb3d41d487d86365829e9eaf9a6] 2024-11-17T08:15:01,928 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting df59cc85272a48318542421eb21c0b0d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731831296571 2024-11-17T08:15:01,928 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b33139bad554f7fa4442dd018ed2b9a, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731831296571 2024-11-17T08:15:01,929 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 51e7a59911804f44b6f252f29ef4a9df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731831297227 2024-11-17T08:15:01,929 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 652503f7b0864a8796c5c4fac9f9ca04, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731831297227 2024-11-17T08:15:01,929 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 525863c91fea452587a57482c149f887, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731831299417 2024-11-17T08:15:01,929 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 508cedb3d41d487d86365829e9eaf9a6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731831299417 2024-11-17T08:15:01,935 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:15:01,937 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8e9eb0dab895bf5f9f5249707e9712d#B#compaction#147 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:01,937 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/26baf3e0f83c4945913dd179fbd812c6 is 50, key is test_row_0/B:col10/1731831299417/Put/seqid=0 2024-11-17T08:15:01,938 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411175e4c75a752544b16a4910e74bbf04555_e8e9eb0dab895bf5f9f5249707e9712d store=[table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:15:01,940 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411175e4c75a752544b16a4910e74bbf04555_e8e9eb0dab895bf5f9f5249707e9712d, store=[table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:15:01,940 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411175e4c75a752544b16a4910e74bbf04555_e8e9eb0dab895bf5f9f5249707e9712d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:15:01,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742000_1176 (size=12595) 2024-11-17T08:15:01,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742001_1177 (size=4469) 2024-11-17T08:15:01,964 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/26baf3e0f83c4945913dd179fbd812c6 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/26baf3e0f83c4945913dd179fbd812c6 2024-11-17T08:15:01,968 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8e9eb0dab895bf5f9f5249707e9712d/B of e8e9eb0dab895bf5f9f5249707e9712d into 26baf3e0f83c4945913dd179fbd812c6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:01,968 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8e9eb0dab895bf5f9f5249707e9712d: 2024-11-17T08:15:01,968 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., storeName=e8e9eb0dab895bf5f9f5249707e9712d/B, priority=13, startTime=1731831301926; duration=0sec 2024-11-17T08:15:01,968 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:15:01,969 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8e9eb0dab895bf5f9f5249707e9712d:B 2024-11-17T08:15:01,969 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:15:01,970 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:15:01,970 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): e8e9eb0dab895bf5f9f5249707e9712d/C is initiating minor compaction (all files) 2024-11-17T08:15:01,970 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8e9eb0dab895bf5f9f5249707e9712d/C in TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:01,970 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/d964cdbe661d4f309880c19e824e4633, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/3d738c8631314f489057c5dbe1f15c49, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/d65fb2fd06eb44ebb7aefcaacf4d051a] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp, totalSize=35.9 K 2024-11-17T08:15:01,970 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting d964cdbe661d4f309880c19e824e4633, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731831296571 2024-11-17T08:15:01,971 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d738c8631314f489057c5dbe1f15c49, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731831297227 2024-11-17T08:15:01,971 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting d65fb2fd06eb44ebb7aefcaacf4d051a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731831299417 2024-11-17T08:15:01,979 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8e9eb0dab895bf5f9f5249707e9712d#C#compaction#149 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:01,979 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/1008831d42f4499d968107c2d759fafd is 50, key is test_row_0/C:col10/1731831299417/Put/seqid=0 2024-11-17T08:15:01,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742002_1178 (size=12595) 2024-11-17T08:15:02,000 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/1008831d42f4499d968107c2d759fafd as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/1008831d42f4499d968107c2d759fafd 2024-11-17T08:15:02,006 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8e9eb0dab895bf5f9f5249707e9712d/C of e8e9eb0dab895bf5f9f5249707e9712d into 1008831d42f4499d968107c2d759fafd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:02,006 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8e9eb0dab895bf5f9f5249707e9712d: 2024-11-17T08:15:02,006 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., storeName=e8e9eb0dab895bf5f9f5249707e9712d/C, priority=13, startTime=1731831301926; duration=0sec 2024-11-17T08:15:02,006 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:02,006 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8e9eb0dab895bf5f9f5249707e9712d:C 2024-11-17T08:15:02,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T08:15:02,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:02,047 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing e8e9eb0dab895bf5f9f5249707e9712d 3/3 column families, dataSize=154.31 KB heapSize=426 KB 2024-11-17T08:15:02,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=A 2024-11-17T08:15:02,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:15:02,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=B 2024-11-17T08:15:02,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:15:02,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=C 2024-11-17T08:15:02,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:15:02,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117db8c783fe98545aabf399b35da9d0f84_e8e9eb0dab895bf5f9f5249707e9712d is 50, key is test_row_0/A:col10/1731831300618/Put/seqid=0 2024-11-17T08:15:02,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742003_1179 (size=12304) 2024-11-17T08:15:02,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T08:15:02,353 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8e9eb0dab895bf5f9f5249707e9712d#A#compaction#148 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:02,353 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/6edae9242f1f43c7808c5af80efe2557 is 175, key is test_row_0/A:col10/1731831299417/Put/seqid=0 2024-11-17T08:15:02,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742004_1180 (size=31549) 2024-11-17T08:15:02,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:02,467 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117db8c783fe98545aabf399b35da9d0f84_e8e9eb0dab895bf5f9f5249707e9712d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117db8c783fe98545aabf399b35da9d0f84_e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:15:02,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/38eacde424ca4f22804980da611ef012, store: [table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:15:02,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/38eacde424ca4f22804980da611ef012 is 175, key is test_row_0/A:col10/1731831300618/Put/seqid=0 2024-11-17T08:15:02,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742005_1181 (size=31105) 2024-11-17T08:15:02,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T08:15:02,762 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/6edae9242f1f43c7808c5af80efe2557 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/6edae9242f1f43c7808c5af80efe2557 2024-11-17T08:15:02,767 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8e9eb0dab895bf5f9f5249707e9712d/A of e8e9eb0dab895bf5f9f5249707e9712d into 6edae9242f1f43c7808c5af80efe2557(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:02,767 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8e9eb0dab895bf5f9f5249707e9712d: 2024-11-17T08:15:02,767 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., storeName=e8e9eb0dab895bf5f9f5249707e9712d/A, priority=13, startTime=1731831301926; duration=0sec 2024-11-17T08:15:02,767 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:02,767 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8e9eb0dab895bf5f9f5249707e9712d:A 2024-11-17T08:15:02,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:15:02,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. as already flushing 2024-11-17T08:15:02,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:02,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831362789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:02,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:02,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:02,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831362790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:02,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:02,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:02,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:02,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:02,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:02,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:02,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831362794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:02,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:02,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831362794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:02,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:02,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831362794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:02,794 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e3d0adb87c01,40631,1731831247805 for 'test_row_2' in e8e9eb0dab895bf5f9f5249707e9712d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T08:15:02,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:02,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:02,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:02,795 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to e3d0adb87c01,40631,1731831247805 for 'test_row_0' in e8e9eb0dab895bf5f9f5249707e9712d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T08:15:02,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:02,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:02,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:02,796 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to e3d0adb87c01,40631,1731831247805 for 'test_row_1' in e8e9eb0dab895bf5f9f5249707e9712d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T08:15:02,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:02,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:02,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:02,874 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=231, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/38eacde424ca4f22804980da611ef012 2024-11-17T08:15:02,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/9b9a39d2872e4082a10a4233df1ae38e is 50, key is test_row_0/B:col10/1731831300618/Put/seqid=0 2024-11-17T08:15:02,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742006_1182 (size=12151) 2024-11-17T08:15:02,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:02,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831362904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:02,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:02,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:02,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:02,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831362904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:02,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:02,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:02,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:02,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:03,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:03,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831363114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:03,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:03,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:03,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831363114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:03,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:03,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:03,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:03,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:03,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:03,294 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/9b9a39d2872e4082a10a4233df1ae38e 2024-11-17T08:15:03,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/faa02a5239aa41f68c83d04add83ff13 is 50, key is test_row_0/C:col10/1731831300618/Put/seqid=0 2024-11-17T08:15:03,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742007_1183 (size=12151) 2024-11-17T08:15:03,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:03,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831363423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:03,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:03,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831363423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:03,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:03,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:03,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:03,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:03,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:03,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:03,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T08:15:03,706 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/faa02a5239aa41f68c83d04add83ff13 2024-11-17T08:15:03,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/38eacde424ca4f22804980da611ef012 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/38eacde424ca4f22804980da611ef012 2024-11-17T08:15:03,714 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/38eacde424ca4f22804980da611ef012, entries=150, sequenceid=231, filesize=30.4 K 2024-11-17T08:15:03,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/9b9a39d2872e4082a10a4233df1ae38e as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/9b9a39d2872e4082a10a4233df1ae38e 2024-11-17T08:15:03,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,719 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/9b9a39d2872e4082a10a4233df1ae38e, entries=150, sequenceid=231, filesize=11.9 K 2024-11-17T08:15:03,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/faa02a5239aa41f68c83d04add83ff13 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/faa02a5239aa41f68c83d04add83ff13 2024-11-17T08:15:03,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,734 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/faa02a5239aa41f68c83d04add83ff13, entries=150, sequenceid=231, filesize=11.9 K 2024-11-17T08:15:03,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,735 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~425.95 KB/436176, currentSize=33.54 KB/34350 for e8e9eb0dab895bf5f9f5249707e9712d in 1688ms, sequenceid=231, compaction requested=false 2024-11-17T08:15:03,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for e8e9eb0dab895bf5f9f5249707e9712d: 2024-11-17T08:15:03,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:03,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-17T08:15:03,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-17T08:15:03,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,740 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-17T08:15:03,740 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3010 sec 2024-11-17T08:15:03,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,742 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.3070 sec 2024-11-17T08:15:03,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:15:03,938 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e8e9eb0dab895bf5f9f5249707e9712d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T08:15:03,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=A 2024-11-17T08:15:03,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:03,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=B 2024-11-17T08:15:03,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:03,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=C 2024-11-17T08:15:03,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:03,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172ec44ff8ed854857b8b43a42b6baa1b8_e8e9eb0dab895bf5f9f5249707e9712d is 50, key is test_row_0/A:col10/1731831303937/Put/seqid=0 2024-11-17T08:15:03,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742008_1184 (size=14794) 2024-11-17T08:15:03,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:03,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731831303979, max timestamp=1731831303994Num uniques 859; 2024-11-17T08:15:03,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T08:15:03,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731831303979, max timestamp=1731831303994Num uniques 859; 2024-11-17T08:15:03,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T08:15:03,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731831303979, max timestamp=1731831303994Num uniques 859; 2024-11-17T08:15:03,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T08:15:04,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:04,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831364034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:04,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:04,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:04,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:04,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:04,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831364037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:04,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:04,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:04,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:04,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:04,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831364144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:04,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:04,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:04,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:04,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:04,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831364153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:04,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:04,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:04,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:04,355 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:04,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:04,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831364354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:04,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:04,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:04,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:04,358 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172ec44ff8ed854857b8b43a42b6baa1b8_e8e9eb0dab895bf5f9f5249707e9712d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172ec44ff8ed854857b8b43a42b6baa1b8_e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:15:04,359 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/d9b031540bba4e1495f5eaf0709e56be, store: [table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:15:04,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/d9b031540bba4e1495f5eaf0709e56be is 175, key is test_row_0/A:col10/1731831303937/Put/seqid=0 2024-11-17T08:15:04,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:04,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831364364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:04,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:04,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:04,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:04,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742009_1185 (size=39745) 2024-11-17T08:15:04,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:04,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831364663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:04,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:04,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:04,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:04,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:04,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831364674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:04,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:04,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:04,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:04,776 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=242, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/d9b031540bba4e1495f5eaf0709e56be 2024-11-17T08:15:04,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/af5392041dc649c78be9855ff8efdb18 is 50, key is test_row_0/B:col10/1731831303937/Put/seqid=0 2024-11-17T08:15:04,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742010_1186 (size=9757) 2024-11-17T08:15:04,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:04,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831364804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:04,807 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to e3d0adb87c01,40631,1731831247805 for 'test_row_0' in e8e9eb0dab895bf5f9f5249707e9712d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T08:15:04,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:04,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:04,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:04,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:04,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831364814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:04,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:04,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831364814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:04,818 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e3d0adb87c01,40631,1731831247805 for 'test_row_2' in e8e9eb0dab895bf5f9f5249707e9712d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T08:15:04,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:04,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:04,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:04,819 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to e3d0adb87c01,40631,1731831247805 for 'test_row_1' in e8e9eb0dab895bf5f9f5249707e9712d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T08:15:04,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:04,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:04,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:05,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:05,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831365184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:05,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:05,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:05,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:05,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:05,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831365184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:05,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:05,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:05,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:05,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/af5392041dc649c78be9855ff8efdb18 2024-11-17T08:15:05,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/e7fcf3ddcc2b450893abb82555f2d325 is 50, key is test_row_0/C:col10/1731831303937/Put/seqid=0 2024-11-17T08:15:05,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742011_1187 (size=9757) 2024-11-17T08:15:05,202 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/e7fcf3ddcc2b450893abb82555f2d325 2024-11-17T08:15:05,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/d9b031540bba4e1495f5eaf0709e56be as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/d9b031540bba4e1495f5eaf0709e56be 2024-11-17T08:15:05,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/d9b031540bba4e1495f5eaf0709e56be, entries=200, sequenceid=242, filesize=38.8 K 2024-11-17T08:15:05,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/af5392041dc649c78be9855ff8efdb18 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/af5392041dc649c78be9855ff8efdb18 2024-11-17T08:15:05,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/af5392041dc649c78be9855ff8efdb18, entries=100, sequenceid=242, filesize=9.5 K 2024-11-17T08:15:05,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/e7fcf3ddcc2b450893abb82555f2d325 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/e7fcf3ddcc2b450893abb82555f2d325 2024-11-17T08:15:05,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/e7fcf3ddcc2b450893abb82555f2d325, entries=100, sequenceid=242, filesize=9.5 K 2024-11-17T08:15:05,225 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for e8e9eb0dab895bf5f9f5249707e9712d in 1287ms, sequenceid=242, compaction requested=true 2024-11-17T08:15:05,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e8e9eb0dab895bf5f9f5249707e9712d: 2024-11-17T08:15:05,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8e9eb0dab895bf5f9f5249707e9712d:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:15:05,225 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:15:05,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:05,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8e9eb0dab895bf5f9f5249707e9712d:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:15:05,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:15:05,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8e9eb0dab895bf5f9f5249707e9712d:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:15:05,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T08:15:05,225 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:15:05,226 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:15:05,226 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): e8e9eb0dab895bf5f9f5249707e9712d/A is initiating minor compaction (all files) 2024-11-17T08:15:05,226 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8e9eb0dab895bf5f9f5249707e9712d/A in TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:05,226 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/6edae9242f1f43c7808c5af80efe2557, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/38eacde424ca4f22804980da611ef012, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/d9b031540bba4e1495f5eaf0709e56be] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp, totalSize=100.0 K 2024-11-17T08:15:05,226 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:05,226 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. files: [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/6edae9242f1f43c7808c5af80efe2557, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/38eacde424ca4f22804980da611ef012, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/d9b031540bba4e1495f5eaf0709e56be] 2024-11-17T08:15:05,227 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:15:05,227 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6edae9242f1f43c7808c5af80efe2557, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731831299417 2024-11-17T08:15:05,227 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): e8e9eb0dab895bf5f9f5249707e9712d/B is initiating minor compaction (all files) 2024-11-17T08:15:05,227 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8e9eb0dab895bf5f9f5249707e9712d/B in TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:05,227 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/26baf3e0f83c4945913dd179fbd812c6, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/9b9a39d2872e4082a10a4233df1ae38e, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/af5392041dc649c78be9855ff8efdb18] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp, totalSize=33.7 K 2024-11-17T08:15:05,227 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38eacde424ca4f22804980da611ef012, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731831300608 2024-11-17T08:15:05,227 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 26baf3e0f83c4945913dd179fbd812c6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731831299417 2024-11-17T08:15:05,228 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9b031540bba4e1495f5eaf0709e56be, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731831302782 2024-11-17T08:15:05,230 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b9a39d2872e4082a10a4233df1ae38e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731831300608 2024-11-17T08:15:05,231 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting af5392041dc649c78be9855ff8efdb18, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731831303936 2024-11-17T08:15:05,238 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:15:05,239 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8e9eb0dab895bf5f9f5249707e9712d#B#compaction#157 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:05,240 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/6dabe3ea162a4e68b9ab7608ffea243f is 50, key is test_row_0/B:col10/1731831303937/Put/seqid=0 2024-11-17T08:15:05,241 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411173bff87f7150b45cd9c8154ec925a4467_e8e9eb0dab895bf5f9f5249707e9712d store=[table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:15:05,243 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411173bff87f7150b45cd9c8154ec925a4467_e8e9eb0dab895bf5f9f5249707e9712d, store=[table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:15:05,243 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411173bff87f7150b45cd9c8154ec925a4467_e8e9eb0dab895bf5f9f5249707e9712d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:15:05,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742012_1188 (size=12697) 2024-11-17T08:15:05,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742013_1189 (size=4469) 2024-11-17T08:15:05,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T08:15:05,603 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:15:05,604 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:15:05,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-17T08:15:05,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T08:15:05,606 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:15:05,606 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:15:05,606 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:15:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T08:15:05,722 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8e9eb0dab895bf5f9f5249707e9712d#A#compaction#156 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:05,722 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/3356053fd89e4f61b9cd07a0c65db742 is 175, key is test_row_0/A:col10/1731831303937/Put/seqid=0 2024-11-17T08:15:05,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742014_1190 (size=31758) 2024-11-17T08:15:05,732 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/6dabe3ea162a4e68b9ab7608ffea243f as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/6dabe3ea162a4e68b9ab7608ffea243f 2024-11-17T08:15:05,740 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/3356053fd89e4f61b9cd07a0c65db742 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/3356053fd89e4f61b9cd07a0c65db742 2024-11-17T08:15:05,748 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8e9eb0dab895bf5f9f5249707e9712d/B of e8e9eb0dab895bf5f9f5249707e9712d into 6dabe3ea162a4e68b9ab7608ffea243f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:05,748 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8e9eb0dab895bf5f9f5249707e9712d: 2024-11-17T08:15:05,748 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., storeName=e8e9eb0dab895bf5f9f5249707e9712d/B, priority=13, startTime=1731831305225; duration=0sec 2024-11-17T08:15:05,748 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:15:05,748 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8e9eb0dab895bf5f9f5249707e9712d:B 2024-11-17T08:15:05,748 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:15:05,752 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:15:05,752 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): e8e9eb0dab895bf5f9f5249707e9712d/C is initiating minor compaction (all files) 2024-11-17T08:15:05,752 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8e9eb0dab895bf5f9f5249707e9712d/C in TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:05,752 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/1008831d42f4499d968107c2d759fafd, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/faa02a5239aa41f68c83d04add83ff13, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/e7fcf3ddcc2b450893abb82555f2d325] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp, totalSize=33.7 K 2024-11-17T08:15:05,753 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 1008831d42f4499d968107c2d759fafd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731831299417 2024-11-17T08:15:05,755 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting faa02a5239aa41f68c83d04add83ff13, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731831300608 2024-11-17T08:15:05,756 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting e7fcf3ddcc2b450893abb82555f2d325, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731831303936 2024-11-17T08:15:05,757 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8e9eb0dab895bf5f9f5249707e9712d/A of e8e9eb0dab895bf5f9f5249707e9712d into 3356053fd89e4f61b9cd07a0c65db742(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:05,757 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8e9eb0dab895bf5f9f5249707e9712d: 2024-11-17T08:15:05,757 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., storeName=e8e9eb0dab895bf5f9f5249707e9712d/A, priority=13, startTime=1731831305225; duration=0sec 2024-11-17T08:15:05,757 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:05,757 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8e9eb0dab895bf5f9f5249707e9712d:A 2024-11-17T08:15:05,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-17T08:15:05,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:05,759 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing e8e9eb0dab895bf5f9f5249707e9712d 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-17T08:15:05,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=A 2024-11-17T08:15:05,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:15:05,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=B 2024-11-17T08:15:05,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:15:05,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=C 2024-11-17T08:15:05,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:15:05,782 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8e9eb0dab895bf5f9f5249707e9712d#C#compaction#158 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:05,783 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/72206f5cef194a8c835762ac5c306bfd is 50, key is test_row_0/C:col10/1731831303937/Put/seqid=0 2024-11-17T08:15:05,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a8c9423486c64c47be0e403cb308b72e_e8e9eb0dab895bf5f9f5249707e9712d is 50, key is test_row_0/A:col10/1731831304032/Put/seqid=0 2024-11-17T08:15:05,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742015_1191 (size=12697) 2024-11-17T08:15:05,806 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/72206f5cef194a8c835762ac5c306bfd as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/72206f5cef194a8c835762ac5c306bfd 2024-11-17T08:15:05,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742016_1192 (size=12454) 2024-11-17T08:15:05,815 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8e9eb0dab895bf5f9f5249707e9712d/C of e8e9eb0dab895bf5f9f5249707e9712d into 72206f5cef194a8c835762ac5c306bfd(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:05,815 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8e9eb0dab895bf5f9f5249707e9712d: 2024-11-17T08:15:05,815 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., storeName=e8e9eb0dab895bf5f9f5249707e9712d/C, priority=13, startTime=1731831305225; duration=0sec 2024-11-17T08:15:05,815 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:05,815 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8e9eb0dab895bf5f9f5249707e9712d:C 2024-11-17T08:15:05,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T08:15:06,110 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T08:15:06,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:15:06,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. as already flushing 2024-11-17T08:15:06,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:06,213 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a8c9423486c64c47be0e403cb308b72e_e8e9eb0dab895bf5f9f5249707e9712d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a8c9423486c64c47be0e403cb308b72e_e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:15:06,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/f1ad1188ef0142279ac044b098c4e495, store: [table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:15:06,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/f1ad1188ef0142279ac044b098c4e495 is 175, key is test_row_0/A:col10/1731831304032/Put/seqid=0 2024-11-17T08:15:06,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742017_1193 (size=31255) 2024-11-17T08:15:06,220 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=280, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/f1ad1188ef0142279ac044b098c4e495 2024-11-17T08:15:06,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831366215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:06,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:06,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:06,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:06,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:06,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831366221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:06,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:06,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:06,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:06,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/482324e436384e9aa53120fb213be8b9 is 50, key is test_row_0/B:col10/1731831304032/Put/seqid=0 2024-11-17T08:15:06,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742018_1194 (size=12301) 2024-11-17T08:15:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T08:15:06,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831366323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:06,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:06,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:06,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:06,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831366333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:06,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:06,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:06,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:06,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:06,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831366534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:06,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:06,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:06,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:06,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:06,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831366543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:06,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:06,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:06,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:06,632 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/482324e436384e9aa53120fb213be8b9 2024-11-17T08:15:06,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/ebf094d1923c4f2591691c765e76d24a is 50, key is test_row_0/C:col10/1731831304032/Put/seqid=0 2024-11-17T08:15:06,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742019_1195 (size=12301) 2024-11-17T08:15:06,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T08:15:06,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:06,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831366844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:06,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:06,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:06,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:06,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:06,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831366854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:06,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:06,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:06,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:07,043 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/ebf094d1923c4f2591691c765e76d24a 2024-11-17T08:15:07,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/f1ad1188ef0142279ac044b098c4e495 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/f1ad1188ef0142279ac044b098c4e495 2024-11-17T08:15:07,051 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/f1ad1188ef0142279ac044b098c4e495, entries=150, sequenceid=280, filesize=30.5 K 2024-11-17T08:15:07,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/482324e436384e9aa53120fb213be8b9 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/482324e436384e9aa53120fb213be8b9 2024-11-17T08:15:07,056 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/482324e436384e9aa53120fb213be8b9, entries=150, sequenceid=280, filesize=12.0 K 2024-11-17T08:15:07,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/ebf094d1923c4f2591691c765e76d24a as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/ebf094d1923c4f2591691c765e76d24a 2024-11-17T08:15:07,061 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/ebf094d1923c4f2591691c765e76d24a, entries=150, sequenceid=280, filesize=12.0 K 2024-11-17T08:15:07,062 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for e8e9eb0dab895bf5f9f5249707e9712d in 1303ms, sequenceid=280, compaction requested=false 2024-11-17T08:15:07,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for e8e9eb0dab895bf5f9f5249707e9712d: 2024-11-17T08:15:07,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:07,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-17T08:15:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-17T08:15:07,065 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-17T08:15:07,065 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4570 sec 2024-11-17T08:15:07,066 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.4610 sec 2024-11-17T08:15:07,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:15:07,355 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e8e9eb0dab895bf5f9f5249707e9712d 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-17T08:15:07,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=A 2024-11-17T08:15:07,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:07,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=B 2024-11-17T08:15:07,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:07,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=C 2024-11-17T08:15:07,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:07,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117bb3c37dae8d24b67b768c8bcd2c52ddf_e8e9eb0dab895bf5f9f5249707e9712d is 50, key is test_row_0/A:col10/1731831306219/Put/seqid=0 2024-11-17T08:15:07,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742020_1196 (size=14994) 2024-11-17T08:15:07,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:07,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831367398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:07,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:07,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:07,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:07,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731831307373, max timestamp=1731831307402Num uniques 859; 2024-11-17T08:15:07,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T08:15:07,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731831307373, max timestamp=1731831307402Num uniques 859; 2024-11-17T08:15:07,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T08:15:07,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731831307373, max timestamp=1731831307402Num uniques 859; 2024-11-17T08:15:07,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T08:15:07,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:07,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831367469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:07,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:07,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:07,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:07,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:07,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831367514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:07,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:07,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:07,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:07,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:07,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831367583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:07,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:07,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:07,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:07,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:07,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831367734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:07,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:07,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:07,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:07,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T08:15:07,753 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:15:07,754 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:15:07,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-17T08:15:07,755 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:15:07,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-17T08:15:07,756 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:15:07,756 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:15:07,780 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:07,783 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117bb3c37dae8d24b67b768c8bcd2c52ddf_e8e9eb0dab895bf5f9f5249707e9712d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117bb3c37dae8d24b67b768c8bcd2c52ddf_e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:15:07,784 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/ff2886849de04744afca152d47c9751a, store: [table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:15:07,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/ff2886849de04744afca152d47c9751a is 175, key is test_row_0/A:col10/1731831306219/Put/seqid=0 2024-11-17T08:15:07,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742021_1197 (size=39949) 2024-11-17T08:15:07,789 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=292, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/ff2886849de04744afca152d47c9751a 2024-11-17T08:15:07,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:07,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831367793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:07,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:07,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:07,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:07,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/5bc78f2de7c448e59ec44f1b22ff7377 is 50, key is test_row_0/B:col10/1731831306219/Put/seqid=0 2024-11-17T08:15:07,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742022_1198 (size=12301) 2024-11-17T08:15:07,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-17T08:15:07,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-17T08:15:07,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:07,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. as already flushing 2024-11-17T08:15:07,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:07,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:07,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:07,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:08,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:08,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831368043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:08,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:08,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:08,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:08,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-17T08:15:08,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:08,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. as already flushing 2024-11-17T08:15:08,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:08,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:08,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:08,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:08,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-17T08:15:08,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:08,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831368103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:08,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:08,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:08,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:08,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/5bc78f2de7c448e59ec44f1b22ff7377 2024-11-17T08:15:08,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/969c87b1ee624b968a4b8b4722b3a433 is 50, key is test_row_0/C:col10/1731831306219/Put/seqid=0 2024-11-17T08:15:08,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742023_1199 (size=12301) 2024-11-17T08:15:08,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-17T08:15:08,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:08,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. as already flushing 2024-11-17T08:15:08,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:08,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:08,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:08,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:08,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-17T08:15:08,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:08,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. as already flushing 2024-11-17T08:15:08,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:08,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:08,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:08,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:08,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-17T08:15:08,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-17T08:15:08,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:08,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. as already flushing 2024-11-17T08:15:08,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:08,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:08,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:08,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:08,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:08,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831368554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:08,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:08,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:08,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:08,612 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/969c87b1ee624b968a4b8b4722b3a433 2024-11-17T08:15:08,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/ff2886849de04744afca152d47c9751a as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/ff2886849de04744afca152d47c9751a 2024-11-17T08:15:08,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:08,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831368613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:08,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:08,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:08,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:08,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/ff2886849de04744afca152d47c9751a, entries=200, sequenceid=292, filesize=39.0 K 2024-11-17T08:15:08,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/5bc78f2de7c448e59ec44f1b22ff7377 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/5bc78f2de7c448e59ec44f1b22ff7377 2024-11-17T08:15:08,625 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/5bc78f2de7c448e59ec44f1b22ff7377, entries=150, sequenceid=292, filesize=12.0 K 2024-11-17T08:15:08,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/969c87b1ee624b968a4b8b4722b3a433 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/969c87b1ee624b968a4b8b4722b3a433 2024-11-17T08:15:08,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/969c87b1ee624b968a4b8b4722b3a433, entries=150, sequenceid=292, filesize=12.0 K 2024-11-17T08:15:08,630 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for e8e9eb0dab895bf5f9f5249707e9712d in 1275ms, sequenceid=292, compaction requested=true 2024-11-17T08:15:08,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e8e9eb0dab895bf5f9f5249707e9712d: 2024-11-17T08:15:08,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8e9eb0dab895bf5f9f5249707e9712d:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:15:08,630 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:15:08,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:08,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8e9eb0dab895bf5f9f5249707e9712d:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:15:08,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:08,631 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:15:08,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8e9eb0dab895bf5f9f5249707e9712d:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:15:08,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:15:08,631 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:15:08,631 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102962 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:15:08,631 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): e8e9eb0dab895bf5f9f5249707e9712d/B is initiating minor compaction (all files) 2024-11-17T08:15:08,631 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): e8e9eb0dab895bf5f9f5249707e9712d/A is initiating minor compaction (all files) 2024-11-17T08:15:08,631 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8e9eb0dab895bf5f9f5249707e9712d/B in TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:08,631 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8e9eb0dab895bf5f9f5249707e9712d/A in TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:08,632 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/6dabe3ea162a4e68b9ab7608ffea243f, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/482324e436384e9aa53120fb213be8b9, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/5bc78f2de7c448e59ec44f1b22ff7377] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp, totalSize=36.4 K 2024-11-17T08:15:08,632 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/3356053fd89e4f61b9cd07a0c65db742, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/f1ad1188ef0142279ac044b098c4e495, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/ff2886849de04744afca152d47c9751a] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp, totalSize=100.5 K 2024-11-17T08:15:08,632 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:08,632 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. files: [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/3356053fd89e4f61b9cd07a0c65db742, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/f1ad1188ef0142279ac044b098c4e495, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/ff2886849de04744afca152d47c9751a] 2024-11-17T08:15:08,632 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 6dabe3ea162a4e68b9ab7608ffea243f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731831300614 2024-11-17T08:15:08,632 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3356053fd89e4f61b9cd07a0c65db742, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731831300614 2024-11-17T08:15:08,632 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 482324e436384e9aa53120fb213be8b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731831304010 2024-11-17T08:15:08,632 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1ad1188ef0142279ac044b098c4e495, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731831304010 2024-11-17T08:15:08,632 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bc78f2de7c448e59ec44f1b22ff7377, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731831306206 2024-11-17T08:15:08,633 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff2886849de04744afca152d47c9751a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731831306206 2024-11-17T08:15:08,638 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:15:08,640 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8e9eb0dab895bf5f9f5249707e9712d#B#compaction#165 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:08,640 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/92cbfc95123e4a07918933e72504fa5a is 50, key is test_row_0/B:col10/1731831306219/Put/seqid=0 2024-11-17T08:15:08,642 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117610850ccb5e94fca853fe662b400b007_e8e9eb0dab895bf5f9f5249707e9712d store=[table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:15:08,643 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117610850ccb5e94fca853fe662b400b007_e8e9eb0dab895bf5f9f5249707e9712d, store=[table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:15:08,644 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117610850ccb5e94fca853fe662b400b007_e8e9eb0dab895bf5f9f5249707e9712d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:15:08,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742024_1200 (size=12949) 2024-11-17T08:15:08,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742025_1201 (size=4469) 2024-11-17T08:15:08,651 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/92cbfc95123e4a07918933e72504fa5a as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/92cbfc95123e4a07918933e72504fa5a 2024-11-17T08:15:08,656 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8e9eb0dab895bf5f9f5249707e9712d/B of e8e9eb0dab895bf5f9f5249707e9712d into 92cbfc95123e4a07918933e72504fa5a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:08,656 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8e9eb0dab895bf5f9f5249707e9712d: 2024-11-17T08:15:08,656 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., storeName=e8e9eb0dab895bf5f9f5249707e9712d/B, priority=13, startTime=1731831308630; duration=0sec 2024-11-17T08:15:08,656 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:15:08,656 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8e9eb0dab895bf5f9f5249707e9712d:B 2024-11-17T08:15:08,656 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:15:08,657 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:15:08,657 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): e8e9eb0dab895bf5f9f5249707e9712d/C is initiating minor compaction (all files) 2024-11-17T08:15:08,657 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8e9eb0dab895bf5f9f5249707e9712d/C in TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:08,657 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/72206f5cef194a8c835762ac5c306bfd, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/ebf094d1923c4f2591691c765e76d24a, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/969c87b1ee624b968a4b8b4722b3a433] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp, totalSize=36.4 K 2024-11-17T08:15:08,657 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 72206f5cef194a8c835762ac5c306bfd, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1731831300614 2024-11-17T08:15:08,658 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting ebf094d1923c4f2591691c765e76d24a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731831304010 2024-11-17T08:15:08,658 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 969c87b1ee624b968a4b8b4722b3a433, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731831306206 2024-11-17T08:15:08,666 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8e9eb0dab895bf5f9f5249707e9712d#C#compaction#167 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:08,667 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/34f83707252e4937844046c6a498681f is 50, key is test_row_0/C:col10/1731831306219/Put/seqid=0 2024-11-17T08:15:08,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-17T08:15:08,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:08,671 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing e8e9eb0dab895bf5f9f5249707e9712d 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-17T08:15:08,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=A 2024-11-17T08:15:08,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:15:08,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=B 2024-11-17T08:15:08,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:15:08,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=C 2024-11-17T08:15:08,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:15:08,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742026_1202 (size=12949) 2024-11-17T08:15:08,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111711cec3490ac2430895c31cc08a2710f2_e8e9eb0dab895bf5f9f5249707e9712d is 50, key is test_row_0/A:col10/1731831307432/Put/seqid=0 2024-11-17T08:15:08,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742027_1203 (size=12454) 2024-11-17T08:15:08,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:15:08,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. as already flushing 2024-11-17T08:15:08,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:08,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831368852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:08,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:08,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:08,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:08,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831368853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:08,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:08,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:08,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:08,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:08,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:08,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831368855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:08,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:08,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:08,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:08,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-17T08:15:08,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:08,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:08,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831368963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:08,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831368963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:08,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:08,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:08,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:08,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831368963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:08,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:08,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:08,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:08,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:08,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:08,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:08,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:09,049 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8e9eb0dab895bf5f9f5249707e9712d#A#compaction#166 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:09,049 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/70e27e63d5f34f9c8b31fec5168c1970 is 175, key is test_row_0/A:col10/1731831306219/Put/seqid=0 2024-11-17T08:15:09,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742028_1204 (size=31903) 2024-11-17T08:15:09,083 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/34f83707252e4937844046c6a498681f as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/34f83707252e4937844046c6a498681f 2024-11-17T08:15:09,088 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8e9eb0dab895bf5f9f5249707e9712d/C of e8e9eb0dab895bf5f9f5249707e9712d into 34f83707252e4937844046c6a498681f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:09,088 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8e9eb0dab895bf5f9f5249707e9712d: 2024-11-17T08:15:09,088 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., storeName=e8e9eb0dab895bf5f9f5249707e9712d/C, priority=13, startTime=1731831308631; duration=0sec 2024-11-17T08:15:09,088 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:09,088 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8e9eb0dab895bf5f9f5249707e9712d:C 2024-11-17T08:15:09,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:09,101 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111711cec3490ac2430895c31cc08a2710f2_e8e9eb0dab895bf5f9f5249707e9712d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111711cec3490ac2430895c31cc08a2710f2_e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:15:09,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/46e1f3a936384b47948aa8bbcb8ef9c0, store: [table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:15:09,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/46e1f3a936384b47948aa8bbcb8ef9c0 is 175, key is test_row_0/A:col10/1731831307432/Put/seqid=0 2024-11-17T08:15:09,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742029_1205 (size=31255) 2024-11-17T08:15:09,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:09,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831369173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:09,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:09,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:09,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:09,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:09,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831369173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:09,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:09,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:09,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:09,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:09,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831369174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:09,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:09,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:09,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:09,462 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/70e27e63d5f34f9c8b31fec5168c1970 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/70e27e63d5f34f9c8b31fec5168c1970 2024-11-17T08:15:09,467 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8e9eb0dab895bf5f9f5249707e9712d/A of e8e9eb0dab895bf5f9f5249707e9712d into 70e27e63d5f34f9c8b31fec5168c1970(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:09,467 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8e9eb0dab895bf5f9f5249707e9712d: 2024-11-17T08:15:09,467 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., storeName=e8e9eb0dab895bf5f9f5249707e9712d/A, priority=13, startTime=1731831308630; duration=0sec 2024-11-17T08:15:09,467 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:09,467 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8e9eb0dab895bf5f9f5249707e9712d:A 2024-11-17T08:15:09,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:09,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831369484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:09,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:09,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:09,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:09,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831369484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:09,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831369484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:09,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:09,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:09,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:09,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:09,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:09,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:09,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:09,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:09,507 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=328, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/46e1f3a936384b47948aa8bbcb8ef9c0 2024-11-17T08:15:09,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/1dcb2d03ccca4afe95d100eeeeb348b2 is 50, key is test_row_0/B:col10/1731831307432/Put/seqid=0 2024-11-17T08:15:09,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742030_1206 (size=12301) 2024-11-17T08:15:09,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:09,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44138 deadline: 1731831369574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:09,579 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e3d0adb87c01,40631,1731831247805 for 'test_row_2' in e8e9eb0dab895bf5f9f5249707e9712d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T08:15:09,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:09,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:09,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:09,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:09,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44080 deadline: 1731831369633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:09,635 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to e3d0adb87c01,40631,1731831247805 for 'test_row_2' in e8e9eb0dab895bf5f9f5249707e9712d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T08:15:09,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:09,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:09,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:09,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-17T08:15:09,919 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/1dcb2d03ccca4afe95d100eeeeb348b2 2024-11-17T08:15:09,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/f90735da6f474372950b1d7b41fd319e is 50, key is test_row_0/C:col10/1731831307432/Put/seqid=0 2024-11-17T08:15:09,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742031_1207 (size=12301) 2024-11-17T08:15:09,930 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/f90735da6f474372950b1d7b41fd319e 2024-11-17T08:15:09,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/46e1f3a936384b47948aa8bbcb8ef9c0 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/46e1f3a936384b47948aa8bbcb8ef9c0 2024-11-17T08:15:09,938 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/46e1f3a936384b47948aa8bbcb8ef9c0, entries=150, sequenceid=328, filesize=30.5 K 2024-11-17T08:15:09,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/1dcb2d03ccca4afe95d100eeeeb348b2 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/1dcb2d03ccca4afe95d100eeeeb348b2 2024-11-17T08:15:09,943 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/1dcb2d03ccca4afe95d100eeeeb348b2, entries=150, sequenceid=328, filesize=12.0 K 2024-11-17T08:15:09,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/f90735da6f474372950b1d7b41fd319e as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/f90735da6f474372950b1d7b41fd319e 2024-11-17T08:15:09,949 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/f90735da6f474372950b1d7b41fd319e, entries=150, sequenceid=328, filesize=12.0 K 2024-11-17T08:15:09,950 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for e8e9eb0dab895bf5f9f5249707e9712d in 1278ms, sequenceid=328, compaction requested=false 2024-11-17T08:15:09,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for e8e9eb0dab895bf5f9f5249707e9712d: 2024-11-17T08:15:09,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:09,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-17T08:15:09,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-17T08:15:09,953 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-17T08:15:09,953 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1950 sec 2024-11-17T08:15:09,955 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 2.2000 sec 2024-11-17T08:15:09,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:15:09,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e8e9eb0dab895bf5f9f5249707e9712d 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-17T08:15:09,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=A 2024-11-17T08:15:09,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:09,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=B 2024-11-17T08:15:09,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:09,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=C 2024-11-17T08:15:09,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:10,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411178cb9d0d954cd4e25b807a8eabd03e9fb_e8e9eb0dab895bf5f9f5249707e9712d is 50, key is test_row_0/A:col10/1731831308837/Put/seqid=0 2024-11-17T08:15:10,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742032_1208 (size=20074) 2024-11-17T08:15:10,009 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:10,013 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411178cb9d0d954cd4e25b807a8eabd03e9fb_e8e9eb0dab895bf5f9f5249707e9712d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178cb9d0d954cd4e25b807a8eabd03e9fb_e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:15:10,014 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/a97f3362856441c5ad8228dd87ee32c9, store: [table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:15:10,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/a97f3362856441c5ad8228dd87ee32c9 is 175, key is test_row_0/A:col10/1731831308837/Put/seqid=0 2024-11-17T08:15:10,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742033_1209 (size=57333) 2024-11-17T08:15:10,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:10,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831370021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:10,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:10,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:10,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831370022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:10,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:10,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:10,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:10,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:10,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:10,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:10,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831370026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:10,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:10,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:10,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:10,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:10,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831370133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:10,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:10,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:10,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:10,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:10,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831370134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:10,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:10,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:10,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:10,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:10,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831370134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:10,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:10,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:10,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:10,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:10,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831370343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:10,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:10,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831370344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:10,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:10,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:10,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:10,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:10,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:10,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:10,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:10,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831370344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:10,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:10,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:10,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:10,422 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=344, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/a97f3362856441c5ad8228dd87ee32c9 2024-11-17T08:15:10,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/1b53b1a7db604301ba720ea92aec93ac is 50, key is test_row_0/B:col10/1731831308837/Put/seqid=0 2024-11-17T08:15:10,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742034_1210 (size=12301) 2024-11-17T08:15:10,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:10,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831370654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:10,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:10,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:10,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:10,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:10,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831370654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:10,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:10,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:10,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:10,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:10,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831370654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:10,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:10,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:10,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:10,681 INFO [Thread-638 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-638. 2024-11-17T08:15:10,681 DEBUG [Thread-638 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:15:10,681 DEBUG [Thread-638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:10,681 DEBUG [Thread-638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:10,681 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:15:10,682 INFO [Thread-636 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-636. 2024-11-17T08:15:10,682 DEBUG [Thread-636 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:15:10,682 DEBUG [Thread-636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:10,682 DEBUG [Thread-636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:10,682 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:15:10,683 INFO [Thread-634 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-634. 2024-11-17T08:15:10,683 DEBUG [Thread-634 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:15:10,683 DEBUG [Thread-634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:10,683 DEBUG [Thread-634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:10,683 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:15:10,689 INFO [Thread-640 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-640. 2024-11-17T08:15:10,689 DEBUG [Thread-640 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:15:10,689 DEBUG [Thread-640 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:10,689 DEBUG [Thread-640 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:10,689 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:15:10,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/1b53b1a7db604301ba720ea92aec93ac 2024-11-17T08:15:10,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/76cb1bd10e2c4e4490cc746b64a8b2df is 50, key is test_row_0/C:col10/1731831308837/Put/seqid=0 2024-11-17T08:15:10,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742035_1211 (size=12301) 2024-11-17T08:15:11,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:11,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:11,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44088 deadline: 1731831371165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:11,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:11,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44112 deadline: 1731831371165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:11,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44100 deadline: 1731831371165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:11,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:11,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:11,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:11,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:11,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:11,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:11,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:11,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:11,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:11,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/76cb1bd10e2c4e4490cc746b64a8b2df 2024-11-17T08:15:11,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/a97f3362856441c5ad8228dd87ee32c9 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/a97f3362856441c5ad8228dd87ee32c9 2024-11-17T08:15:11,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/a97f3362856441c5ad8228dd87ee32c9, entries=300, sequenceid=344, filesize=56.0 K 2024-11-17T08:15:11,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/1b53b1a7db604301ba720ea92aec93ac as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/1b53b1a7db604301ba720ea92aec93ac 2024-11-17T08:15:11,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/1b53b1a7db604301ba720ea92aec93ac, entries=150, sequenceid=344, filesize=12.0 K 2024-11-17T08:15:11,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/76cb1bd10e2c4e4490cc746b64a8b2df as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/76cb1bd10e2c4e4490cc746b64a8b2df 2024-11-17T08:15:11,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/76cb1bd10e2c4e4490cc746b64a8b2df, entries=150, sequenceid=344, filesize=12.0 K 2024-11-17T08:15:11,341 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for e8e9eb0dab895bf5f9f5249707e9712d in 1345ms, sequenceid=344, compaction requested=true 2024-11-17T08:15:11,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e8e9eb0dab895bf5f9f5249707e9712d: 2024-11-17T08:15:11,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8e9eb0dab895bf5f9f5249707e9712d:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:15:11,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:11,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8e9eb0dab895bf5f9f5249707e9712d:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:15:11,341 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:15:11,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:11,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e8e9eb0dab895bf5f9f5249707e9712d:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:15:11,342 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:15:11,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:15:11,343 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:15:11,343 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 120491 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:15:11,343 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): e8e9eb0dab895bf5f9f5249707e9712d/B is initiating minor compaction (all files) 2024-11-17T08:15:11,343 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): e8e9eb0dab895bf5f9f5249707e9712d/A is initiating minor compaction (all files) 2024-11-17T08:15:11,343 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8e9eb0dab895bf5f9f5249707e9712d/A in TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:11,343 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8e9eb0dab895bf5f9f5249707e9712d/B in TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:11,343 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/70e27e63d5f34f9c8b31fec5168c1970, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/46e1f3a936384b47948aa8bbcb8ef9c0, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/a97f3362856441c5ad8228dd87ee32c9] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp, totalSize=117.7 K 2024-11-17T08:15:11,343 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/92cbfc95123e4a07918933e72504fa5a, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/1dcb2d03ccca4afe95d100eeeeb348b2, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/1b53b1a7db604301ba720ea92aec93ac] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp, totalSize=36.7 K 2024-11-17T08:15:11,343 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:11,343 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. files: [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/70e27e63d5f34f9c8b31fec5168c1970, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/46e1f3a936384b47948aa8bbcb8ef9c0, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/a97f3362856441c5ad8228dd87ee32c9] 2024-11-17T08:15:11,343 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92cbfc95123e4a07918933e72504fa5a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731831306206 2024-11-17T08:15:11,343 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 70e27e63d5f34f9c8b31fec5168c1970, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731831306206 2024-11-17T08:15:11,343 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1dcb2d03ccca4afe95d100eeeeb348b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731831307432 2024-11-17T08:15:11,344 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 46e1f3a936384b47948aa8bbcb8ef9c0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731831307432 2024-11-17T08:15:11,344 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b53b1a7db604301ba720ea92aec93ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731831308837 2024-11-17T08:15:11,344 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting a97f3362856441c5ad8228dd87ee32c9, keycount=300, bloomtype=ROW, size=56.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731831308837 2024-11-17T08:15:11,351 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8e9eb0dab895bf5f9f5249707e9712d#B#compaction#174 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:11,351 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/7f50b2bd2eec43748efc461f7eb4823c is 50, key is test_row_0/B:col10/1731831308837/Put/seqid=0 2024-11-17T08:15:11,351 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:15:11,354 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117b8a535940f06491a9422bb08c375d2a7_e8e9eb0dab895bf5f9f5249707e9712d store=[table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:15:11,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742036_1212 (size=13051) 2024-11-17T08:15:11,367 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117b8a535940f06491a9422bb08c375d2a7_e8e9eb0dab895bf5f9f5249707e9712d, store=[table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:15:11,367 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b8a535940f06491a9422bb08c375d2a7_e8e9eb0dab895bf5f9f5249707e9712d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:15:11,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742037_1213 (size=4469) 2024-11-17T08:15:11,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:15:11,607 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e8e9eb0dab895bf5f9f5249707e9712d 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-17T08:15:11,608 INFO [Thread-625 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-625. 2024-11-17T08:15:11,608 DEBUG [Thread-625 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:15:11,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=A 2024-11-17T08:15:11,608 DEBUG [Thread-625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:11,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:11,608 DEBUG [Thread-625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:11,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=B 2024-11-17T08:15:11,608 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:15:11,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:11,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=C 2024-11-17T08:15:11,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:11,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d9a67ff8f2e04871ac501c52b076beb0_e8e9eb0dab895bf5f9f5249707e9712d is 50, key is test_row_0/A:col10/1731831310014/Put/seqid=0 2024-11-17T08:15:11,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742038_1214 (size=12454) 2024-11-17T08:15:11,666 INFO [Thread-627 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-627. 2024-11-17T08:15:11,666 DEBUG [Thread-627 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:15:11,666 DEBUG [Thread-627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:11,666 DEBUG [Thread-627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:11,666 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:15:11,768 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/7f50b2bd2eec43748efc461f7eb4823c as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/7f50b2bd2eec43748efc461f7eb4823c 2024-11-17T08:15:11,772 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8e9eb0dab895bf5f9f5249707e9712d#A#compaction#175 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:11,773 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/8c2aadc9180346ddbcf2e3fbcad7b658 is 175, key is test_row_0/A:col10/1731831308837/Put/seqid=0 2024-11-17T08:15:11,775 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8e9eb0dab895bf5f9f5249707e9712d/B of e8e9eb0dab895bf5f9f5249707e9712d into 7f50b2bd2eec43748efc461f7eb4823c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:11,775 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8e9eb0dab895bf5f9f5249707e9712d: 2024-11-17T08:15:11,775 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., storeName=e8e9eb0dab895bf5f9f5249707e9712d/B, priority=13, startTime=1731831311341; duration=0sec 2024-11-17T08:15:11,775 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:15:11,775 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8e9eb0dab895bf5f9f5249707e9712d:B 2024-11-17T08:15:11,775 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:15:11,777 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:15:11,777 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): e8e9eb0dab895bf5f9f5249707e9712d/C is initiating minor compaction (all files) 2024-11-17T08:15:11,777 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e8e9eb0dab895bf5f9f5249707e9712d/C in TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:11,777 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/34f83707252e4937844046c6a498681f, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/f90735da6f474372950b1d7b41fd319e, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/76cb1bd10e2c4e4490cc746b64a8b2df] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp, totalSize=36.7 K 2024-11-17T08:15:11,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742039_1215 (size=32005) 2024-11-17T08:15:11,778 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34f83707252e4937844046c6a498681f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731831306206 2024-11-17T08:15:11,778 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting f90735da6f474372950b1d7b41fd319e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731831307432 2024-11-17T08:15:11,779 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76cb1bd10e2c4e4490cc746b64a8b2df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731831308837 2024-11-17T08:15:11,788 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e8e9eb0dab895bf5f9f5249707e9712d#C#compaction#177 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:11,788 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/1252e18c05db4508969ccd3e8e290bba is 50, key is test_row_0/C:col10/1731831308837/Put/seqid=0 2024-11-17T08:15:11,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742040_1216 (size=13051) 2024-11-17T08:15:11,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-17T08:15:11,925 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:15:12,025 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:12,031 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d9a67ff8f2e04871ac501c52b076beb0_e8e9eb0dab895bf5f9f5249707e9712d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d9a67ff8f2e04871ac501c52b076beb0_e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:15:12,032 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/8b9570c89fc4485baef03b605582f02e, store: [table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:15:12,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/8b9570c89fc4485baef03b605582f02e is 175, key is test_row_0/A:col10/1731831310014/Put/seqid=0 2024-11-17T08:15:12,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742041_1217 (size=31255) 2024-11-17T08:15:12,179 INFO [Thread-629 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-629. 2024-11-17T08:15:12,179 DEBUG [Thread-629 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:15:12,179 DEBUG [Thread-629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:12,179 INFO [Thread-631 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-631. 2024-11-17T08:15:12,179 DEBUG [Thread-631 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:15:12,179 DEBUG [Thread-631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:12,179 DEBUG [Thread-629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:12,179 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:15:12,180 DEBUG [Thread-631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:12,180 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:15:12,186 INFO [Thread-623 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-623. 2024-11-17T08:15:12,186 DEBUG [Thread-623 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:15:12,186 DEBUG [Thread-623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:12,186 DEBUG [Thread-623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:12,186 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:15:12,187 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T08:15:12,187 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-17T08:15:12,187 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-17T08:15:12,187 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-11-17T08:15:12,187 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-17T08:15:12,187 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-17T08:15:12,187 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T08:15:12,187 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3054 2024-11-17T08:15:12,187 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2979 2024-11-17T08:15:12,187 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T08:15:12,187 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1274 2024-11-17T08:15:12,187 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3821 rows 2024-11-17T08:15:12,187 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1281 2024-11-17T08:15:12,187 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3838 rows 2024-11-17T08:15:12,187 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T08:15:12,187 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T08:15:12,187 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:12,187 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/8c2aadc9180346ddbcf2e3fbcad7b658 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/8c2aadc9180346ddbcf2e3fbcad7b658 2024-11-17T08:15:12,187 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:12,187 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:15:12,188 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-17T08:15:12,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T08:15:12,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-17T08:15:12,192 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731831312191"}]},"ts":"1731831312191"} 2024-11-17T08:15:12,193 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8e9eb0dab895bf5f9f5249707e9712d/A of e8e9eb0dab895bf5f9f5249707e9712d into 8c2aadc9180346ddbcf2e3fbcad7b658(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:12,193 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8e9eb0dab895bf5f9f5249707e9712d: 2024-11-17T08:15:12,193 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., storeName=e8e9eb0dab895bf5f9f5249707e9712d/A, priority=13, startTime=1731831311341; duration=0sec 2024-11-17T08:15:12,193 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:12,193 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8e9eb0dab895bf5f9f5249707e9712d:A 2024-11-17T08:15:12,194 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T08:15:12,194 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T08:15:12,195 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T08:15:12,196 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e8e9eb0dab895bf5f9f5249707e9712d, UNASSIGN}] 2024-11-17T08:15:12,197 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e8e9eb0dab895bf5f9f5249707e9712d, UNASSIGN 2024-11-17T08:15:12,197 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=e8e9eb0dab895bf5f9f5249707e9712d, regionState=CLOSING, regionLocation=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:12,198 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/1252e18c05db4508969ccd3e8e290bba as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/1252e18c05db4508969ccd3e8e290bba 2024-11-17T08:15:12,199 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e8e9eb0dab895bf5f9f5249707e9712d, UNASSIGN because future has completed 2024-11-17T08:15:12,199 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T08:15:12,199 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805}] 2024-11-17T08:15:12,203 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e8e9eb0dab895bf5f9f5249707e9712d/C of e8e9eb0dab895bf5f9f5249707e9712d into 1252e18c05db4508969ccd3e8e290bba(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:12,203 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e8e9eb0dab895bf5f9f5249707e9712d: 2024-11-17T08:15:12,203 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., storeName=e8e9eb0dab895bf5f9f5249707e9712d/C, priority=13, startTime=1731831311342; duration=0sec 2024-11-17T08:15:12,204 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:12,204 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e8e9eb0dab895bf5f9f5249707e9712d:C 2024-11-17T08:15:12,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-17T08:15:12,353 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(122): Close e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:15:12,353 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T08:15:12,353 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1722): Closing e8e9eb0dab895bf5f9f5249707e9712d, disabling compactions & flushes 2024-11-17T08:15:12,354 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:12,439 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=363, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/8b9570c89fc4485baef03b605582f02e 2024-11-17T08:15:12,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/512172f83cb043b39c7a6f62fd8ca411 is 50, key is test_row_0/B:col10/1731831310014/Put/seqid=0 2024-11-17T08:15:12,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742042_1218 (size=12301) 2024-11-17T08:15:12,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-17T08:15:12,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-17T08:15:12,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/512172f83cb043b39c7a6f62fd8ca411 2024-11-17T08:15:12,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/43e4b6106e7742259c79d0524871ea5a is 50, key is test_row_0/C:col10/1731831310014/Put/seqid=0 2024-11-17T08:15:12,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742043_1219 (size=12301) 2024-11-17T08:15:13,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/43e4b6106e7742259c79d0524871ea5a 2024-11-17T08:15:13,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/8b9570c89fc4485baef03b605582f02e as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/8b9570c89fc4485baef03b605582f02e 2024-11-17T08:15:13,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/8b9570c89fc4485baef03b605582f02e, entries=150, sequenceid=363, filesize=30.5 K 2024-11-17T08:15:13,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/512172f83cb043b39c7a6f62fd8ca411 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/512172f83cb043b39c7a6f62fd8ca411 2024-11-17T08:15:13,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/512172f83cb043b39c7a6f62fd8ca411, entries=150, sequenceid=363, filesize=12.0 K 2024-11-17T08:15:13,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/43e4b6106e7742259c79d0524871ea5a as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/43e4b6106e7742259c79d0524871ea5a 2024-11-17T08:15:13,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/43e4b6106e7742259c79d0524871ea5a, entries=150, sequenceid=363, filesize=12.0 K 2024-11-17T08:15:13,306 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=26.84 KB/27480 for e8e9eb0dab895bf5f9f5249707e9712d in 1699ms, sequenceid=363, compaction requested=false 2024-11-17T08:15:13,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e8e9eb0dab895bf5f9f5249707e9712d: 2024-11-17T08:15:13,307 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:13,307 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:13,307 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. after waiting 0 ms 2024-11-17T08:15:13,307 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:13,307 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(2902): Flushing e8e9eb0dab895bf5f9f5249707e9712d 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-17T08:15:13,307 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=A 2024-11-17T08:15:13,307 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:13,307 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=B 2024-11-17T08:15:13,307 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:13,307 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e8e9eb0dab895bf5f9f5249707e9712d, store=C 2024-11-17T08:15:13,307 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:13,315 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117efc72112cd574c21b101ff4084eb799e_e8e9eb0dab895bf5f9f5249707e9712d is 50, key is test_row_0/A:col10/1731831312175/Put/seqid=0 2024-11-17T08:15:13,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742044_1220 (size=12454) 2024-11-17T08:15:13,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-17T08:15:13,721 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:13,731 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117efc72112cd574c21b101ff4084eb799e_e8e9eb0dab895bf5f9f5249707e9712d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117efc72112cd574c21b101ff4084eb799e_e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:15:13,732 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/f4625955585046dfa8745b75ab24ce06, store: [table=TestAcidGuarantees family=A region=e8e9eb0dab895bf5f9f5249707e9712d] 2024-11-17T08:15:13,733 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/f4625955585046dfa8745b75ab24ce06 is 175, key is test_row_0/A:col10/1731831312175/Put/seqid=0 2024-11-17T08:15:13,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742045_1221 (size=31255) 2024-11-17T08:15:14,137 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=373, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/f4625955585046dfa8745b75ab24ce06 2024-11-17T08:15:14,144 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/a623a75aa6674f8ca7855d2a75dface0 is 50, key is test_row_0/B:col10/1731831312175/Put/seqid=0 2024-11-17T08:15:14,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742046_1222 (size=12301) 2024-11-17T08:15:14,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-17T08:15:14,549 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/a623a75aa6674f8ca7855d2a75dface0 2024-11-17T08:15:14,563 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/56f8be69878048e1a62b775a41362a2f is 50, key is test_row_0/C:col10/1731831312175/Put/seqid=0 2024-11-17T08:15:14,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742047_1223 (size=12301) 2024-11-17T08:15:14,969 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/56f8be69878048e1a62b775a41362a2f 2024-11-17T08:15:14,979 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/A/f4625955585046dfa8745b75ab24ce06 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/f4625955585046dfa8745b75ab24ce06 2024-11-17T08:15:14,983 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/f4625955585046dfa8745b75ab24ce06, entries=150, sequenceid=373, filesize=30.5 K 2024-11-17T08:15:14,984 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/B/a623a75aa6674f8ca7855d2a75dface0 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/a623a75aa6674f8ca7855d2a75dface0 2024-11-17T08:15:14,988 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/a623a75aa6674f8ca7855d2a75dface0, entries=150, sequenceid=373, filesize=12.0 K 2024-11-17T08:15:14,989 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/.tmp/C/56f8be69878048e1a62b775a41362a2f as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/56f8be69878048e1a62b775a41362a2f 2024-11-17T08:15:14,993 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/56f8be69878048e1a62b775a41362a2f, entries=150, sequenceid=373, filesize=12.0 K 2024-11-17T08:15:14,994 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for e8e9eb0dab895bf5f9f5249707e9712d in 1687ms, sequenceid=373, compaction requested=true 2024-11-17T08:15:14,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/d9356a37f801441e84c1577c0e921067, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/035678266982435f8d6ba4d2579d6ee2, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/8bbbc2dea6984a3893124f53ecb1a4af, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/af11922a4ebe4482b198ce37e26b2451, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/9ded661498cc4563a13c4dc2538b2678, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/502e212023934fcbbb8db7e5a9b75e07, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/8859163ffe6f42bb98ff04bf2dbad874, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/3fccda8059494b97a3b5fd25c8e73a10, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/3f7fbd1842fb4fcd83b6ad5fea8c02b1, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/cd3807996e1e459a83b82c9455b07620, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/0b33139bad554f7fa4442dd018ed2b9a, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/652503f7b0864a8796c5c4fac9f9ca04, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/6edae9242f1f43c7808c5af80efe2557, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/508cedb3d41d487d86365829e9eaf9a6, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/38eacde424ca4f22804980da611ef012, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/d9b031540bba4e1495f5eaf0709e56be, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/3356053fd89e4f61b9cd07a0c65db742, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/f1ad1188ef0142279ac044b098c4e495, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/ff2886849de04744afca152d47c9751a, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/70e27e63d5f34f9c8b31fec5168c1970, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/46e1f3a936384b47948aa8bbcb8ef9c0, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/a97f3362856441c5ad8228dd87ee32c9] to archive 2024-11-17T08:15:14,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T08:15:14,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/d9356a37f801441e84c1577c0e921067 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/d9356a37f801441e84c1577c0e921067 2024-11-17T08:15:14,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/035678266982435f8d6ba4d2579d6ee2 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/035678266982435f8d6ba4d2579d6ee2 2024-11-17T08:15:14,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/8bbbc2dea6984a3893124f53ecb1a4af to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/8bbbc2dea6984a3893124f53ecb1a4af 2024-11-17T08:15:15,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/af11922a4ebe4482b198ce37e26b2451 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/af11922a4ebe4482b198ce37e26b2451 2024-11-17T08:15:15,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/9ded661498cc4563a13c4dc2538b2678 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/9ded661498cc4563a13c4dc2538b2678 2024-11-17T08:15:15,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/502e212023934fcbbb8db7e5a9b75e07 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/502e212023934fcbbb8db7e5a9b75e07 2024-11-17T08:15:15,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/8859163ffe6f42bb98ff04bf2dbad874 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/8859163ffe6f42bb98ff04bf2dbad874 2024-11-17T08:15:15,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/3fccda8059494b97a3b5fd25c8e73a10 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/3fccda8059494b97a3b5fd25c8e73a10 2024-11-17T08:15:15,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/3f7fbd1842fb4fcd83b6ad5fea8c02b1 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/3f7fbd1842fb4fcd83b6ad5fea8c02b1 2024-11-17T08:15:15,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/cd3807996e1e459a83b82c9455b07620 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/cd3807996e1e459a83b82c9455b07620 2024-11-17T08:15:15,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/0b33139bad554f7fa4442dd018ed2b9a to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/0b33139bad554f7fa4442dd018ed2b9a 2024-11-17T08:15:15,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/652503f7b0864a8796c5c4fac9f9ca04 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/652503f7b0864a8796c5c4fac9f9ca04 2024-11-17T08:15:15,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/6edae9242f1f43c7808c5af80efe2557 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/6edae9242f1f43c7808c5af80efe2557 2024-11-17T08:15:15,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/508cedb3d41d487d86365829e9eaf9a6 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/508cedb3d41d487d86365829e9eaf9a6 2024-11-17T08:15:15,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/38eacde424ca4f22804980da611ef012 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/38eacde424ca4f22804980da611ef012 2024-11-17T08:15:15,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/d9b031540bba4e1495f5eaf0709e56be to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/d9b031540bba4e1495f5eaf0709e56be 2024-11-17T08:15:15,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/3356053fd89e4f61b9cd07a0c65db742 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/3356053fd89e4f61b9cd07a0c65db742 2024-11-17T08:15:15,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/f1ad1188ef0142279ac044b098c4e495 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/f1ad1188ef0142279ac044b098c4e495 2024-11-17T08:15:15,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/ff2886849de04744afca152d47c9751a to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/ff2886849de04744afca152d47c9751a 2024-11-17T08:15:15,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/70e27e63d5f34f9c8b31fec5168c1970 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/70e27e63d5f34f9c8b31fec5168c1970 2024-11-17T08:15:15,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/46e1f3a936384b47948aa8bbcb8ef9c0 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/46e1f3a936384b47948aa8bbcb8ef9c0 2024-11-17T08:15:15,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/a97f3362856441c5ad8228dd87ee32c9 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/a97f3362856441c5ad8228dd87ee32c9 2024-11-17T08:15:15,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/d33819c99464452cbdf3e97c74c65e94, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/52920993ae8c409b83231c9bab4f8da5, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/6f978e293b8340c7af15278aeb362224, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/561d7b256cb44f28a300929a896c231a, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/4fb8f9c508174e65ad76e2dc8e5cb127, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/a25b870fe4084e74a29cd43fb29283ba, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/1ba51ab6777f4c9b830035819380b36a, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/aa5dab037cb346db9495e56978eec622, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/b7e9b9ee9a374c4bae78f3a145f0966e, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/df59cc85272a48318542421eb21c0b0d, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/65bf1b6f20854dad8d26c3e460b97386, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/51e7a59911804f44b6f252f29ef4a9df, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/26baf3e0f83c4945913dd179fbd812c6, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/525863c91fea452587a57482c149f887, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/9b9a39d2872e4082a10a4233df1ae38e, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/6dabe3ea162a4e68b9ab7608ffea243f, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/af5392041dc649c78be9855ff8efdb18, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/482324e436384e9aa53120fb213be8b9, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/92cbfc95123e4a07918933e72504fa5a, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/5bc78f2de7c448e59ec44f1b22ff7377, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/1dcb2d03ccca4afe95d100eeeeb348b2, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/1b53b1a7db604301ba720ea92aec93ac] to archive 2024-11-17T08:15:15,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T08:15:15,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/d33819c99464452cbdf3e97c74c65e94 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/d33819c99464452cbdf3e97c74c65e94 2024-11-17T08:15:15,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/52920993ae8c409b83231c9bab4f8da5 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/52920993ae8c409b83231c9bab4f8da5 2024-11-17T08:15:15,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/6f978e293b8340c7af15278aeb362224 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/6f978e293b8340c7af15278aeb362224 2024-11-17T08:15:15,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/561d7b256cb44f28a300929a896c231a to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/561d7b256cb44f28a300929a896c231a 2024-11-17T08:15:15,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/4fb8f9c508174e65ad76e2dc8e5cb127 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/4fb8f9c508174e65ad76e2dc8e5cb127 2024-11-17T08:15:15,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/a25b870fe4084e74a29cd43fb29283ba to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/a25b870fe4084e74a29cd43fb29283ba 2024-11-17T08:15:15,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/1ba51ab6777f4c9b830035819380b36a to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/1ba51ab6777f4c9b830035819380b36a 2024-11-17T08:15:15,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/aa5dab037cb346db9495e56978eec622 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/aa5dab037cb346db9495e56978eec622 2024-11-17T08:15:15,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/b7e9b9ee9a374c4bae78f3a145f0966e to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/b7e9b9ee9a374c4bae78f3a145f0966e 2024-11-17T08:15:15,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/df59cc85272a48318542421eb21c0b0d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/df59cc85272a48318542421eb21c0b0d 2024-11-17T08:15:15,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/65bf1b6f20854dad8d26c3e460b97386 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/65bf1b6f20854dad8d26c3e460b97386 2024-11-17T08:15:15,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/51e7a59911804f44b6f252f29ef4a9df to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/51e7a59911804f44b6f252f29ef4a9df 2024-11-17T08:15:15,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/26baf3e0f83c4945913dd179fbd812c6 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/26baf3e0f83c4945913dd179fbd812c6 2024-11-17T08:15:15,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/525863c91fea452587a57482c149f887 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/525863c91fea452587a57482c149f887 2024-11-17T08:15:15,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/9b9a39d2872e4082a10a4233df1ae38e to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/9b9a39d2872e4082a10a4233df1ae38e 2024-11-17T08:15:15,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/6dabe3ea162a4e68b9ab7608ffea243f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/6dabe3ea162a4e68b9ab7608ffea243f 2024-11-17T08:15:15,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/af5392041dc649c78be9855ff8efdb18 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/af5392041dc649c78be9855ff8efdb18 2024-11-17T08:15:15,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/482324e436384e9aa53120fb213be8b9 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/482324e436384e9aa53120fb213be8b9 2024-11-17T08:15:15,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/92cbfc95123e4a07918933e72504fa5a to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/92cbfc95123e4a07918933e72504fa5a 2024-11-17T08:15:15,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/5bc78f2de7c448e59ec44f1b22ff7377 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/5bc78f2de7c448e59ec44f1b22ff7377 2024-11-17T08:15:15,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/1dcb2d03ccca4afe95d100eeeeb348b2 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/1dcb2d03ccca4afe95d100eeeeb348b2 2024-11-17T08:15:15,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/1b53b1a7db604301ba720ea92aec93ac to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/1b53b1a7db604301ba720ea92aec93ac 2024-11-17T08:15:15,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/bfae669b183f49cea2b6fb32e037b788, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/5580bc6eec3a41d497d77687f580acdc, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/fbe39c2a136f474ab01388d9c0ce7794, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/d656683944ea4588a954c236f733997f, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/a447e26e13e548848a5ab4d1f53c782c, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/c45c99774ed44192a577873cc1c0f131, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/4ae8f1a609154636861125c4c5b4aeec, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/2f57219a05104260afa417a50ecc5cf5, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/18079be5841a4a6bae392310f6b7e0c7, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/d964cdbe661d4f309880c19e824e4633, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/a48393c568624f6dbf3a2c06e669c551, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/3d738c8631314f489057c5dbe1f15c49, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/1008831d42f4499d968107c2d759fafd, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/d65fb2fd06eb44ebb7aefcaacf4d051a, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/faa02a5239aa41f68c83d04add83ff13, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/72206f5cef194a8c835762ac5c306bfd, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/e7fcf3ddcc2b450893abb82555f2d325, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/ebf094d1923c4f2591691c765e76d24a, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/34f83707252e4937844046c6a498681f, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/969c87b1ee624b968a4b8b4722b3a433, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/f90735da6f474372950b1d7b41fd319e, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/76cb1bd10e2c4e4490cc746b64a8b2df] to archive 2024-11-17T08:15:15,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T08:15:15,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/bfae669b183f49cea2b6fb32e037b788 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/bfae669b183f49cea2b6fb32e037b788 2024-11-17T08:15:15,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/5580bc6eec3a41d497d77687f580acdc to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/5580bc6eec3a41d497d77687f580acdc 2024-11-17T08:15:15,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/fbe39c2a136f474ab01388d9c0ce7794 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/fbe39c2a136f474ab01388d9c0ce7794 2024-11-17T08:15:15,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/d656683944ea4588a954c236f733997f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/d656683944ea4588a954c236f733997f 2024-11-17T08:15:15,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/a447e26e13e548848a5ab4d1f53c782c to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/a447e26e13e548848a5ab4d1f53c782c 2024-11-17T08:15:15,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/c45c99774ed44192a577873cc1c0f131 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/c45c99774ed44192a577873cc1c0f131 2024-11-17T08:15:15,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/4ae8f1a609154636861125c4c5b4aeec to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/4ae8f1a609154636861125c4c5b4aeec 2024-11-17T08:15:15,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/2f57219a05104260afa417a50ecc5cf5 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/2f57219a05104260afa417a50ecc5cf5 2024-11-17T08:15:15,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/18079be5841a4a6bae392310f6b7e0c7 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/18079be5841a4a6bae392310f6b7e0c7 2024-11-17T08:15:15,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/d964cdbe661d4f309880c19e824e4633 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/d964cdbe661d4f309880c19e824e4633 2024-11-17T08:15:15,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/a48393c568624f6dbf3a2c06e669c551 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/a48393c568624f6dbf3a2c06e669c551 2024-11-17T08:15:15,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/3d738c8631314f489057c5dbe1f15c49 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/3d738c8631314f489057c5dbe1f15c49 2024-11-17T08:15:15,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/1008831d42f4499d968107c2d759fafd to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/1008831d42f4499d968107c2d759fafd 2024-11-17T08:15:15,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/d65fb2fd06eb44ebb7aefcaacf4d051a to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/d65fb2fd06eb44ebb7aefcaacf4d051a 2024-11-17T08:15:15,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/faa02a5239aa41f68c83d04add83ff13 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/faa02a5239aa41f68c83d04add83ff13 2024-11-17T08:15:15,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/72206f5cef194a8c835762ac5c306bfd to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/72206f5cef194a8c835762ac5c306bfd 2024-11-17T08:15:15,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/e7fcf3ddcc2b450893abb82555f2d325 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/e7fcf3ddcc2b450893abb82555f2d325 2024-11-17T08:15:15,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/ebf094d1923c4f2591691c765e76d24a to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/ebf094d1923c4f2591691c765e76d24a 2024-11-17T08:15:15,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/34f83707252e4937844046c6a498681f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/34f83707252e4937844046c6a498681f 2024-11-17T08:15:15,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/969c87b1ee624b968a4b8b4722b3a433 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/969c87b1ee624b968a4b8b4722b3a433 2024-11-17T08:15:15,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/f90735da6f474372950b1d7b41fd319e to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/f90735da6f474372950b1d7b41fd319e 2024-11-17T08:15:15,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/76cb1bd10e2c4e4490cc746b64a8b2df to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/76cb1bd10e2c4e4490cc746b64a8b2df 2024-11-17T08:15:15,058 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/recovered.edits/376.seqid, newMaxSeqId=376, maxSeqId=4 2024-11-17T08:15:15,058 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. 2024-11-17T08:15:15,058 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1676): Region close journal for e8e9eb0dab895bf5f9f5249707e9712d: Waiting for close lock at 1731831312353Running coprocessor pre-close hooks at 1731831312353Disabling compacts and flushes for region at 1731831312353Disabling writes for close at 1731831313307 (+954 ms)Obtaining lock to block concurrent updates at 1731831313307Preparing flush snapshotting stores in e8e9eb0dab895bf5f9f5249707e9712d at 1731831313307Finished memstore snapshotting TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731831313307Flushing stores of TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d. at 1731831313308 (+1 ms)Flushing e8e9eb0dab895bf5f9f5249707e9712d/A: creating writer at 1731831313308Flushing mob file e8e9eb0dab895bf5f9f5249707e9712d/A: appending metadata at 1731831313314 (+6 ms)Flushing mob file e8e9eb0dab895bf5f9f5249707e9712d/A: closing flushed file at 1731831313314Flushing e8e9eb0dab895bf5f9f5249707e9712d/A: appending metadata at 1731831313732 (+418 ms)Flushing e8e9eb0dab895bf5f9f5249707e9712d/A: closing flushed file at 1731831313732Flushing e8e9eb0dab895bf5f9f5249707e9712d/B: creating writer at 1731831314141 (+409 ms)Flushing e8e9eb0dab895bf5f9f5249707e9712d/B: appending metadata at 1731831314144 (+3 ms)Flushing e8e9eb0dab895bf5f9f5249707e9712d/B: closing flushed file at 1731831314144Flushing e8e9eb0dab895bf5f9f5249707e9712d/C: creating writer at 1731831314559 (+415 ms)Flushing e8e9eb0dab895bf5f9f5249707e9712d/C: appending metadata at 1731831314563 (+4 ms)Flushing e8e9eb0dab895bf5f9f5249707e9712d/C: closing flushed file at 1731831314563Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@379627bd: reopening flushed file at 1731831314978 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74514ae0: reopening flushed file at 1731831314983 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72996f04: reopening flushed file at 1731831314988 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for e8e9eb0dab895bf5f9f5249707e9712d in 1687ms, sequenceid=373, compaction requested=true at 1731831314994 (+6 ms)Writing region close event to WAL at 1731831315055 (+61 ms)Running coprocessor post-close hooks at 1731831315058 (+3 ms)Closed at 1731831315058 2024-11-17T08:15:15,060 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(157): Closed e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:15:15,060 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=e8e9eb0dab895bf5f9f5249707e9712d, regionState=CLOSED 2024-11-17T08:15:15,062 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 because future has completed 2024-11-17T08:15:15,064 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-17T08:15:15,064 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseRegionProcedure e8e9eb0dab895bf5f9f5249707e9712d, server=e3d0adb87c01,40631,1731831247805 in 2.8630 sec 2024-11-17T08:15:15,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-17T08:15:15,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e8e9eb0dab895bf5f9f5249707e9712d, UNASSIGN in 2.8680 sec 2024-11-17T08:15:15,067 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-17T08:15:15,067 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.8700 sec 2024-11-17T08:15:15,068 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731831315068"}]},"ts":"1731831315068"} 2024-11-17T08:15:15,069 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T08:15:15,069 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T08:15:15,070 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.8810 sec 2024-11-17T08:15:16,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-17T08:15:16,374 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:15:16,375 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-17T08:15:16,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T08:15:16,377 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T08:15:16,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-17T08:15:16,378 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=62, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T08:15:16,381 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:15:16,383 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A, FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B, FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C, FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/recovered.edits] 2024-11-17T08:15:16,386 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/8b9570c89fc4485baef03b605582f02e to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/8b9570c89fc4485baef03b605582f02e 2024-11-17T08:15:16,387 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/8c2aadc9180346ddbcf2e3fbcad7b658 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/8c2aadc9180346ddbcf2e3fbcad7b658 2024-11-17T08:15:16,388 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/f4625955585046dfa8745b75ab24ce06 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/A/f4625955585046dfa8745b75ab24ce06 2024-11-17T08:15:16,390 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/512172f83cb043b39c7a6f62fd8ca411 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/512172f83cb043b39c7a6f62fd8ca411 2024-11-17T08:15:16,391 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/7f50b2bd2eec43748efc461f7eb4823c to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/7f50b2bd2eec43748efc461f7eb4823c 2024-11-17T08:15:16,392 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/a623a75aa6674f8ca7855d2a75dface0 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/B/a623a75aa6674f8ca7855d2a75dface0 2024-11-17T08:15:16,394 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/1252e18c05db4508969ccd3e8e290bba to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/1252e18c05db4508969ccd3e8e290bba 2024-11-17T08:15:16,395 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/43e4b6106e7742259c79d0524871ea5a to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/43e4b6106e7742259c79d0524871ea5a 2024-11-17T08:15:16,396 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/56f8be69878048e1a62b775a41362a2f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/C/56f8be69878048e1a62b775a41362a2f 2024-11-17T08:15:16,398 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/recovered.edits/376.seqid to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d/recovered.edits/376.seqid 2024-11-17T08:15:16,399 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:15:16,399 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T08:15:16,399 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T08:15:16,400 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-17T08:15:16,404 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111711cec3490ac2430895c31cc08a2710f2_e8e9eb0dab895bf5f9f5249707e9712d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111711cec3490ac2430895c31cc08a2710f2_e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:15:16,405 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172ec44ff8ed854857b8b43a42b6baa1b8_e8e9eb0dab895bf5f9f5249707e9712d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172ec44ff8ed854857b8b43a42b6baa1b8_e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:15:16,406 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174491ff7f4e9d41c3838429493c156652_e8e9eb0dab895bf5f9f5249707e9712d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174491ff7f4e9d41c3838429493c156652_e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:15:16,407 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111756914b63d1df4d219479a3752a953ca4_e8e9eb0dab895bf5f9f5249707e9712d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111756914b63d1df4d219479a3752a953ca4_e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:15:16,408 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176ffb0b0fc3bb49c9b15f33a10dea712d_e8e9eb0dab895bf5f9f5249707e9712d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176ffb0b0fc3bb49c9b15f33a10dea712d_e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:15:16,409 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111780ba616916f84e0bab8918828709beca_e8e9eb0dab895bf5f9f5249707e9712d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111780ba616916f84e0bab8918828709beca_e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:15:16,410 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178cb9d0d954cd4e25b807a8eabd03e9fb_e8e9eb0dab895bf5f9f5249707e9712d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178cb9d0d954cd4e25b807a8eabd03e9fb_e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:15:16,411 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117985bef6b1fd64212b3f9e46f3d06953e_e8e9eb0dab895bf5f9f5249707e9712d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117985bef6b1fd64212b3f9e46f3d06953e_e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:15:16,412 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a8c9423486c64c47be0e403cb308b72e_e8e9eb0dab895bf5f9f5249707e9712d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a8c9423486c64c47be0e403cb308b72e_e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:15:16,413 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b9d628d21e41486ea93df90ee59e6a8d_e8e9eb0dab895bf5f9f5249707e9712d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b9d628d21e41486ea93df90ee59e6a8d_e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:15:16,414 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117bb3c37dae8d24b67b768c8bcd2c52ddf_e8e9eb0dab895bf5f9f5249707e9712d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117bb3c37dae8d24b67b768c8bcd2c52ddf_e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:15:16,415 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117cb45eae08f1146e29e7d20f781afe997_e8e9eb0dab895bf5f9f5249707e9712d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117cb45eae08f1146e29e7d20f781afe997_e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:15:16,416 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d52e008b6b4340caacb397e2cb3f5b39_e8e9eb0dab895bf5f9f5249707e9712d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d52e008b6b4340caacb397e2cb3f5b39_e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:15:16,418 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d9a67ff8f2e04871ac501c52b076beb0_e8e9eb0dab895bf5f9f5249707e9712d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d9a67ff8f2e04871ac501c52b076beb0_e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:15:16,419 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117db8c783fe98545aabf399b35da9d0f84_e8e9eb0dab895bf5f9f5249707e9712d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117db8c783fe98545aabf399b35da9d0f84_e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:15:16,420 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e030e7a410064a1da01cdcbdb910a85b_e8e9eb0dab895bf5f9f5249707e9712d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e030e7a410064a1da01cdcbdb910a85b_e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:15:16,422 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117efc72112cd574c21b101ff4084eb799e_e8e9eb0dab895bf5f9f5249707e9712d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117efc72112cd574c21b101ff4084eb799e_e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:15:16,423 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f8078879951d4a0b84993017e0b4572b_e8e9eb0dab895bf5f9f5249707e9712d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f8078879951d4a0b84993017e0b4572b_e8e9eb0dab895bf5f9f5249707e9712d 2024-11-17T08:15:16,424 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T08:15:16,426 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=62, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T08:15:16,429 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T08:15:16,431 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T08:15:16,432 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=62, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T08:15:16,432 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T08:15:16,432 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731831316432"}]},"ts":"9223372036854775807"} 2024-11-17T08:15:16,434 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T08:15:16,434 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e8e9eb0dab895bf5f9f5249707e9712d, NAME => 'TestAcidGuarantees,,1731831287427.e8e9eb0dab895bf5f9f5249707e9712d.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T08:15:16,434 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T08:15:16,434 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731831316434"}]},"ts":"9223372036854775807"} 2024-11-17T08:15:16,436 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T08:15:16,436 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=62, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T08:15:16,437 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 61 msec 2024-11-17T08:15:16,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-17T08:15:16,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T08:15:16,484 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:15:16,498 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=251 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2115572639_22 at /127.0.0.1:55030 [Waiting for operation #269] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5720963-478b-ee25-58c5-86f892de1407/cluster_aa2b9c1b-2c01-fdb4-e68c-d41e93bcbc98/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5720963-478b-ee25-58c5-86f892de1407/cluster_aa2b9c1b-2c01-fdb4-e68c-d41e93bcbc98/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-306999293_22 at /127.0.0.1:43694 [Waiting for operation #434] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-306999293_22 at /127.0.0.1:43796 [Waiting for operation #411] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=278 (was 288), ProcessCount=11 (was 11), AvailableMemoryMB=2170 (was 2289) 2024-11-17T08:15:16,506 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=251, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=278, ProcessCount=11, AvailableMemoryMB=2170 2024-11-17T08:15:16,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T08:15:16,507 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T08:15:16,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T08:15:16,509 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T08:15:16,510 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:16,510 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 63 2024-11-17T08:15:16,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-17T08:15:16,511 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T08:15:16,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742048_1224 (size=1061) 2024-11-17T08:15:16,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-17T08:15:16,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-17T08:15:16,920 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85 2024-11-17T08:15:16,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742049_1225 (size=53) 2024-11-17T08:15:17,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-17T08:15:17,326 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T08:15:17,326 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 23525a0a9ca7b911625b0315ffddfcb8, disabling compactions & flushes 2024-11-17T08:15:17,326 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:17,326 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:17,326 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. after waiting 0 ms 2024-11-17T08:15:17,326 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:17,326 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:17,326 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 23525a0a9ca7b911625b0315ffddfcb8: Waiting for close lock at 1731831317326Disabling compacts and flushes for region at 1731831317326Disabling writes for close at 1731831317326Writing region close event to WAL at 1731831317326Closed at 1731831317326 2024-11-17T08:15:17,327 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-17T08:15:17,327 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T08:15:17,328 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731831317327"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731831317327"}]},"ts":"1731831317327"} 2024-11-17T08:15:17,329 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T08:15:17,330 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T08:15:17,330 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731831317330"}]},"ts":"1731831317330"} 2024-11-17T08:15:17,331 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T08:15:17,332 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=23525a0a9ca7b911625b0315ffddfcb8, ASSIGN}] 2024-11-17T08:15:17,333 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=23525a0a9ca7b911625b0315ffddfcb8, ASSIGN 2024-11-17T08:15:17,334 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=23525a0a9ca7b911625b0315ffddfcb8, ASSIGN; state=OFFLINE, location=e3d0adb87c01,40631,1731831247805; forceNewPlan=false, retain=false 2024-11-17T08:15:17,484 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=23525a0a9ca7b911625b0315ffddfcb8, regionState=OPENING, regionLocation=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:17,486 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=23525a0a9ca7b911625b0315ffddfcb8, ASSIGN because future has completed 2024-11-17T08:15:17,486 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805}] 2024-11-17T08:15:17,640 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:17,641 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7752): Opening region: {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} 2024-11-17T08:15:17,641 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 23525a0a9ca7b911625b0315ffddfcb8 2024-11-17T08:15:17,641 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T08:15:17,641 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7794): checking encryption for 23525a0a9ca7b911625b0315ffddfcb8 2024-11-17T08:15:17,641 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7797): checking classloading for 23525a0a9ca7b911625b0315ffddfcb8 2024-11-17T08:15:17,642 INFO [StoreOpener-23525a0a9ca7b911625b0315ffddfcb8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 23525a0a9ca7b911625b0315ffddfcb8 2024-11-17T08:15:17,644 INFO [StoreOpener-23525a0a9ca7b911625b0315ffddfcb8-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T08:15:17,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-17T08:15:17,644 INFO [StoreOpener-23525a0a9ca7b911625b0315ffddfcb8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 23525a0a9ca7b911625b0315ffddfcb8 columnFamilyName A 2024-11-17T08:15:17,644 DEBUG [StoreOpener-23525a0a9ca7b911625b0315ffddfcb8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:17,645 INFO [StoreOpener-23525a0a9ca7b911625b0315ffddfcb8-1 {}] regionserver.HStore(327): Store=23525a0a9ca7b911625b0315ffddfcb8/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T08:15:17,645 INFO [StoreOpener-23525a0a9ca7b911625b0315ffddfcb8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 23525a0a9ca7b911625b0315ffddfcb8 2024-11-17T08:15:17,646 INFO [StoreOpener-23525a0a9ca7b911625b0315ffddfcb8-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T08:15:17,647 INFO [StoreOpener-23525a0a9ca7b911625b0315ffddfcb8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 23525a0a9ca7b911625b0315ffddfcb8 columnFamilyName B 2024-11-17T08:15:17,647 DEBUG [StoreOpener-23525a0a9ca7b911625b0315ffddfcb8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:17,647 INFO [StoreOpener-23525a0a9ca7b911625b0315ffddfcb8-1 {}] regionserver.HStore(327): Store=23525a0a9ca7b911625b0315ffddfcb8/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T08:15:17,647 INFO [StoreOpener-23525a0a9ca7b911625b0315ffddfcb8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 23525a0a9ca7b911625b0315ffddfcb8 2024-11-17T08:15:17,648 INFO [StoreOpener-23525a0a9ca7b911625b0315ffddfcb8-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T08:15:17,649 INFO [StoreOpener-23525a0a9ca7b911625b0315ffddfcb8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 23525a0a9ca7b911625b0315ffddfcb8 columnFamilyName C 2024-11-17T08:15:17,649 DEBUG [StoreOpener-23525a0a9ca7b911625b0315ffddfcb8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:17,649 INFO [StoreOpener-23525a0a9ca7b911625b0315ffddfcb8-1 {}] regionserver.HStore(327): Store=23525a0a9ca7b911625b0315ffddfcb8/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T08:15:17,649 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:17,649 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1038): replaying wal for 23525a0a9ca7b911625b0315ffddfcb8 2024-11-17T08:15:17,650 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8 2024-11-17T08:15:17,650 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8 2024-11-17T08:15:17,651 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1048): stopping wal replay for 23525a0a9ca7b911625b0315ffddfcb8 2024-11-17T08:15:17,651 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1060): Cleaning up temporary data for 23525a0a9ca7b911625b0315ffddfcb8 2024-11-17T08:15:17,652 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T08:15:17,653 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): writing seq id for 23525a0a9ca7b911625b0315ffddfcb8 2024-11-17T08:15:17,656 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T08:15:17,656 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1114): Opened 23525a0a9ca7b911625b0315ffddfcb8; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60666377, jitterRate=-0.096000537276268}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T08:15:17,656 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 23525a0a9ca7b911625b0315ffddfcb8 2024-11-17T08:15:17,657 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1006): Region open journal for 23525a0a9ca7b911625b0315ffddfcb8: Running coprocessor pre-open hook at 1731831317641Writing region info on filesystem at 1731831317641Initializing all the Stores at 1731831317642 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731831317642Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731831317642Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731831317642Cleaning up temporary data from old regions at 1731831317651 (+9 ms)Running coprocessor post-open hooks at 1731831317656 (+5 ms)Region opened successfully at 1731831317657 (+1 ms) 2024-11-17T08:15:17,658 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., pid=65, masterSystemTime=1731831317638 2024-11-17T08:15:17,660 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:17,660 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:17,661 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=23525a0a9ca7b911625b0315ffddfcb8, regionState=OPEN, openSeqNum=2, regionLocation=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:17,662 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 because future has completed 2024-11-17T08:15:17,665 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-17T08:15:17,665 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure 23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 in 177 msec 2024-11-17T08:15:17,667 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-17T08:15:17,667 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=23525a0a9ca7b911625b0315ffddfcb8, ASSIGN in 333 msec 2024-11-17T08:15:17,667 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T08:15:17,668 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731831317667"}]},"ts":"1731831317667"} 2024-11-17T08:15:17,669 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T08:15:17,670 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T08:15:17,671 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1630 sec 2024-11-17T08:15:18,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-17T08:15:18,654 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:15:18,655 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe11fca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:18,655 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:15:18,655 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:15:18,656 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:15:18,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:15:18,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:15:18,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f70ef3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:18,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:15:18,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:15:18,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:18,659 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:15:18,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1db4879f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:18,660 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:15:18,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:15:18,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:15:18,661 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53800, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:15:18,663 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T08:15:18,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is e3d0adb87c01,45089,1731831246985 2024-11-17T08:15:18,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4add4f9a 2024-11-17T08:15:18,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T08:15:18,665 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45648, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T08:15:18,666 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b5b273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:18,666 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:15:18,666 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:15:18,667 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:15:18,667 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:15:18,667 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:15:18,667 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68e81446, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:18,667 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:15:18,667 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:15:18,668 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:18,668 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:15:18,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@255e9f37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:18,669 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1240be3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:18,669 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:15:18,669 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:15:18,670 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:15:18,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:15:18,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:15:18,671 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@508b594a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:18,671 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:15:18,671 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:15:18,671 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:18,672 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45686, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:15:18,672 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5120a8f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:18,674 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885eb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:18,674 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:15:18,674 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:15:18,675 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:15:18,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:15:18,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:15:18,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fd45969, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:18,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:15:18,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:15:18,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:18,676 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45714, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:15:18,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c07db5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:18,677 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692a6227, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:18,677 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:15:18,677 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:15:18,679 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:15:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:15:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:15:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24acf71a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:18,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:15:18,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:15:18,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:18,681 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:15:18,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a7a66c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:18,684 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@128d97bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:18,684 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:15:18,684 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:15:18,685 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:15:18,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:15:18,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:15:18,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@457dcab6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:18,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:15:18,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:15:18,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:18,687 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45754, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:15:18,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@432b06fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:18,692 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626de388, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:18,692 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:15:18,692 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:15:18,693 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:15:18,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:15:18,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:15:18,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@791343a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:18,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:15:18,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:15:18,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:18,695 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:15:18,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ea97aed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:18,696 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a5c877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:18,696 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:15:18,696 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:15:18,697 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:15:18,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:15:18,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:15:18,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@789ad89c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:18,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:15:18,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:15:18,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:18,699 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:15:18,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77f4305f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:18,700 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679f7426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:18,700 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:15:18,700 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:15:18,701 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:15:18,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:15:18,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:15:18,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@446daaf2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:18,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:15:18,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:15:18,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:18,703 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45812, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:15:18,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@337c8e4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:18,704 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b71af22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:18,704 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:15:18,705 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:15:18,705 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:15:18,705 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:15:18,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:15:18,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b0a815a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:18,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:15:18,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:15:18,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:18,707 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45824, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:15:18,707 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@aef15c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:18,708 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@728d338, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:18,708 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:15:18,708 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:15:18,709 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:15:18,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:15:18,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:15:18,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49558701, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:18,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:15:18,710 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:15:18,710 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:18,711 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45840, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:15:18,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7631d9e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:18,712 DEBUG [Thread-1064 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:15:18,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:15:18,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:15:18,714 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53810, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:15:18,720 DEBUG [Thread-1071 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:15:18,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2] 2024-11-17T08:15:18,721 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:15:18,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:15:18,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:15:18,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-17T08:15:18,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-17T08:15:18,724 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:15:18,724 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:15:18,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2] 2024-11-17T08:15:18,728 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:15:18,728 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:15:18,729 DEBUG [Thread-1077 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:15:18,730 DEBUG [Thread-1077 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:15:18,731 DEBUG [Thread-1077 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:15:18,733 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:15:18,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2] 2024-11-17T08:15:18,736 DEBUG [Thread-1073 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:15:18,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:15:18,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:15:18,739 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53826, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:15:18,741 DEBUG [Thread-1079 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:15:18,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2] 2024-11-17T08:15:18,742 DEBUG [Thread-1062 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:15:18,742 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:15:18,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:15:18,743 DEBUG [Thread-1060 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:15:18,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:15:18,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:15:18,743 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:15:18,744 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:15:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 23525a0a9ca7b911625b0315ffddfcb8 2024-11-17T08:15:18,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23525a0a9ca7b911625b0315ffddfcb8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T08:15:18,745 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53846, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:15:18,745 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53860, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:15:18,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=A 2024-11-17T08:15:18,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:18,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=B 2024-11-17T08:15:18,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:18,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=C 2024-11-17T08:15:18,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:18,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2] 2024-11-17T08:15:18,747 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53830, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:15:18,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2] 2024-11-17T08:15:18,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2] 2024-11-17T08:15:18,756 DEBUG [Thread-1068 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:15:18,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:15:18,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:15:18,760 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53874, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:15:18,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2] 2024-11-17T08:15:18,763 DEBUG [Thread-1066 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:15:18,764 DEBUG [Thread-1066 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:15:18,765 DEBUG [Thread-1066 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:15:18,765 DEBUG [Thread-1075 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:15:18,766 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53876, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:15:18,766 DEBUG [Thread-1075 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:15:18,766 DEBUG [Thread-1075 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:15:18,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2] 2024-11-17T08:15:18,769 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53880, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:15:18,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2] 2024-11-17T08:15:18,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731831318769, max timestamp=1731831318786Num uniques 859; 2024-11-17T08:15:18,791 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T08:15:18,792 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=180, cellSize=8230, totalHeapSize=18960, min timestamp=1731831318769, max timestamp=1731831318786Num uniques 859; 2024-11-17T08:15:18,793 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731831318772, max timestamp=1731831318786Num uniques 859; 2024-11-17T08:15:18,792 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39324, new segment data size=8230, suffix heap size=117064,new segment heap size=18960 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=180 2024-11-17T08:15:18,793 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T08:15:18,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/75b633cb4ce64f1faea17a12b70eccf7 is 50, key is test_row_0/A:col10/1731831318742/Put/seqid=0 2024-11-17T08:15:18,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-17T08:15:18,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742050_1226 (size=12001) 2024-11-17T08:15:18,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:18,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:18,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831378860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:18,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831378869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:18,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:18,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:18,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:18,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53876 deadline: 1731831378870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:18,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:18,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:18,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:18,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831378872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:18,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:18,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831378872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:18,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:18,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:18,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:18,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:18,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:18,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:18,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-17T08:15:18,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:18,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. as already flushing 2024-11-17T08:15:18,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:18,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:18,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:18,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:18,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:18,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831378984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:18,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:18,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831378984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:18,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:18,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831378984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:18,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:18,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:18,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831378985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:18,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:18,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:18,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:18,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:18,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:18,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:18,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:18,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:18,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53876 deadline: 1731831378985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:18,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:18,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:18,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:19,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-17T08:15:19,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:19,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. as already flushing 2024-11-17T08:15:19,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:19,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:19,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:19,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:19,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-17T08:15:19,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-17T08:15:19,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:19,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. as already flushing 2024-11-17T08:15:19,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:19,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:19,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:19,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:19,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:19,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831379194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:19,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:19,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:19,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:19,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831379194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:19,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:19,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:19,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:19,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:19,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:19,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831379194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:19,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:19,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:19,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:19,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:19,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831379195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:19,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:19,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:19,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:19,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:19,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53876 deadline: 1731831379196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:19,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:19,249 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/75b633cb4ce64f1faea17a12b70eccf7 2024-11-17T08:15:19,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/42688fda3cd548159111b977f8b0ce55 is 50, key is test_row_0/B:col10/1731831318742/Put/seqid=0 2024-11-17T08:15:19,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742051_1227 (size=12001) 2024-11-17T08:15:19,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/42688fda3cd548159111b977f8b0ce55 2024-11-17T08:15:19,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/0e06d9a598a745f0a7e175a008654198 is 50, key is test_row_0/C:col10/1731831318742/Put/seqid=0 2024-11-17T08:15:19,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742052_1228 (size=12001) 2024-11-17T08:15:19,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/0e06d9a598a745f0a7e175a008654198 2024-11-17T08:15:19,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/75b633cb4ce64f1faea17a12b70eccf7 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/75b633cb4ce64f1faea17a12b70eccf7 2024-11-17T08:15:19,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-17T08:15:19,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:19,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. as already flushing 2024-11-17T08:15:19,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:19,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:19,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:19,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:19,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-17T08:15:19,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/75b633cb4ce64f1faea17a12b70eccf7, entries=150, sequenceid=11, filesize=11.7 K 2024-11-17T08:15:19,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/42688fda3cd548159111b977f8b0ce55 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/42688fda3cd548159111b977f8b0ce55 2024-11-17T08:15:19,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/42688fda3cd548159111b977f8b0ce55, entries=150, sequenceid=11, filesize=11.7 K 2024-11-17T08:15:19,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/0e06d9a598a745f0a7e175a008654198 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/0e06d9a598a745f0a7e175a008654198 2024-11-17T08:15:19,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/0e06d9a598a745f0a7e175a008654198, entries=150, sequenceid=11, filesize=11.7 K 2024-11-17T08:15:19,386 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=137.36 KB/140656 for 23525a0a9ca7b911625b0315ffddfcb8 in 640ms, sequenceid=11, compaction requested=false 2024-11-17T08:15:19,386 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-17T08:15:19,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:19,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-17T08:15:19,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:19,501 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 23525a0a9ca7b911625b0315ffddfcb8 3/3 column families, dataSize=137.36 KB heapSize=392.20 KB 2024-11-17T08:15:19,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=A 2024-11-17T08:15:19,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:15:19,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=B 2024-11-17T08:15:19,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:15:19,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=C 2024-11-17T08:15:19,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:15:19,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 23525a0a9ca7b911625b0315ffddfcb8 2024-11-17T08:15:19,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. as already flushing 2024-11-17T08:15:19,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/6890b7afd8124947a2a8feba3307f071 is 50, key is test_row_0/A:col10/1731831318814/Put/seqid=0 2024-11-17T08:15:19,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:19,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831379510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:19,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:19,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:19,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:19,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:19,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831379510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:19,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:19,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:19,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:19,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:19,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53876 deadline: 1731831379514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:19,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:19,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:19,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:19,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:19,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831379514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:19,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:19,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:19,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:19,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:19,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831379516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:19,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:19,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:19,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:19,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742053_1229 (size=12001) 2024-11-17T08:15:19,553 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45.67 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/6890b7afd8124947a2a8feba3307f071 2024-11-17T08:15:19,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/cdaee554da47438b994bcc686641b0db is 50, key is test_row_0/B:col10/1731831318814/Put/seqid=0 2024-11-17T08:15:19,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742054_1230 (size=12001) 2024-11-17T08:15:19,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:19,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831379624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:19,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:19,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:19,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:19,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831379624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:19,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:19,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:19,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:19,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:19,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:19,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831379624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:19,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:19,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:19,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:19,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:19,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831379633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:19,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:19,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:19,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:19,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:19,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831379834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:19,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:19,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:19,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:19,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:19,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831379834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:19,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:19,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:19,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:19,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:19,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831379834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:19,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:19,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:19,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:19,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:19,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831379843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:19,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:19,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:19,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:19,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-17T08:15:19,982 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/cdaee554da47438b994bcc686641b0db 2024-11-17T08:15:19,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/78a3dbe261a34d94a7758e985f6bc1a7 is 50, key is test_row_0/C:col10/1731831318814/Put/seqid=0 2024-11-17T08:15:19,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742055_1231 (size=12001) 2024-11-17T08:15:20,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:20,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53876 deadline: 1731831380034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:20,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:20,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:20,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:20,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:20,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831380144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:20,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:20,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:20,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:20,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:20,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831380144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:20,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:20,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:20,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:20,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:20,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831380144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:20,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:20,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:20,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:20,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:20,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831380167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:20,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:20,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:20,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:20,401 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/78a3dbe261a34d94a7758e985f6bc1a7 2024-11-17T08:15:20,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/6890b7afd8124947a2a8feba3307f071 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/6890b7afd8124947a2a8feba3307f071 2024-11-17T08:15:20,413 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/6890b7afd8124947a2a8feba3307f071, entries=150, sequenceid=48, filesize=11.7 K 2024-11-17T08:15:20,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/cdaee554da47438b994bcc686641b0db as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/cdaee554da47438b994bcc686641b0db 2024-11-17T08:15:20,422 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/cdaee554da47438b994bcc686641b0db, entries=150, sequenceid=48, filesize=11.7 K 2024-11-17T08:15:20,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/78a3dbe261a34d94a7758e985f6bc1a7 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/78a3dbe261a34d94a7758e985f6bc1a7 2024-11-17T08:15:20,431 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/78a3dbe261a34d94a7758e985f6bc1a7, entries=150, sequenceid=48, filesize=11.7 K 2024-11-17T08:15:20,432 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~137.36 KB/140656, heapSize ~392.15 KB/401560, currentSize=40.25 KB/41220 for 23525a0a9ca7b911625b0315ffddfcb8 in 931ms, sequenceid=48, compaction requested=false 2024-11-17T08:15:20,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:20,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:20,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-17T08:15:20,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-17T08:15:20,436 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-17T08:15:20,437 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7060 sec 2024-11-17T08:15:20,440 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.7150 sec 2024-11-17T08:15:20,581 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T08:15:20,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 23525a0a9ca7b911625b0315ffddfcb8 2024-11-17T08:15:20,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23525a0a9ca7b911625b0315ffddfcb8 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-17T08:15:20,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=A 2024-11-17T08:15:20,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:20,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=B 2024-11-17T08:15:20,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:20,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=C 2024-11-17T08:15:20,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:20,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/2e781ec865144a6b893f9d42e0cd87b7 is 50, key is test_row_0/A:col10/1731831319507/Put/seqid=0 2024-11-17T08:15:20,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742056_1232 (size=12001) 2024-11-17T08:15:20,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:20,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831380697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:20,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:20,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:20,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:20,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831380698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:20,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:20,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:20,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:20,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:20,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:20,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831380698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:20,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:20,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:20,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:20,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:20,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831380700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:20,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:20,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:20,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:20,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:20,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831380813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:20,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:20,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:20,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:20,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:20,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831380813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:20,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:20,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:20,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:20,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831380813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:20,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:20,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:20,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:20,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831380814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:20,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:20,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:20,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:20,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:20,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:20,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-17T08:15:20,873 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:15:20,874 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:15:20,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-17T08:15:20,875 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:15:20,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-17T08:15:20,876 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:15:20,876 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:15:20,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-17T08:15:21,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:21,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831381023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:21,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:21,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831381024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:21,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:21,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:21,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:21,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831381024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:21,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:21,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831381024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:21,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:21,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:21,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-17T08:15:21,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:21,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. as already flushing 2024-11-17T08:15:21,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:21,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:21,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:21,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:21,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:21,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53876 deadline: 1731831381043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:21,046 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to e3d0adb87c01,40631,1731831247805 for 'test_row_2' in 23525a0a9ca7b911625b0315ffddfcb8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T08:15:21,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:21,082 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/2e781ec865144a6b893f9d42e0cd87b7 2024-11-17T08:15:21,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/0f3d1b57feee459da2f9ffd260c2277d is 50, key is test_row_0/B:col10/1731831319507/Put/seqid=0 2024-11-17T08:15:21,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742057_1233 (size=12001) 2024-11-17T08:15:21,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/0f3d1b57feee459da2f9ffd260c2277d 2024-11-17T08:15:21,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/59e5978a22824bdea6796897ddbce226 is 50, key is test_row_0/C:col10/1731831319507/Put/seqid=0 2024-11-17T08:15:21,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742058_1234 (size=12001) 2024-11-17T08:15:21,136 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/59e5978a22824bdea6796897ddbce226 2024-11-17T08:15:21,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/2e781ec865144a6b893f9d42e0cd87b7 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/2e781ec865144a6b893f9d42e0cd87b7 2024-11-17T08:15:21,149 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/2e781ec865144a6b893f9d42e0cd87b7, entries=150, sequenceid=60, filesize=11.7 K 2024-11-17T08:15:21,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/0f3d1b57feee459da2f9ffd260c2277d as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/0f3d1b57feee459da2f9ffd260c2277d 2024-11-17T08:15:21,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/0f3d1b57feee459da2f9ffd260c2277d, entries=150, sequenceid=60, filesize=11.7 K 2024-11-17T08:15:21,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/59e5978a22824bdea6796897ddbce226 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/59e5978a22824bdea6796897ddbce226 2024-11-17T08:15:21,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/59e5978a22824bdea6796897ddbce226, entries=150, sequenceid=60, filesize=11.7 K 2024-11-17T08:15:21,159 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 23525a0a9ca7b911625b0315ffddfcb8 in 499ms, sequenceid=60, compaction requested=true 2024-11-17T08:15:21,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:21,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23525a0a9ca7b911625b0315ffddfcb8:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:15:21,159 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:15:21,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:21,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23525a0a9ca7b911625b0315ffddfcb8:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:15:21,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:21,159 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:15:21,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23525a0a9ca7b911625b0315ffddfcb8:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:15:21,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:15:21,160 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:15:21,160 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:15:21,160 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): 23525a0a9ca7b911625b0315ffddfcb8/B is initiating minor compaction (all files) 2024-11-17T08:15:21,160 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): 23525a0a9ca7b911625b0315ffddfcb8/A is initiating minor compaction (all files) 2024-11-17T08:15:21,160 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23525a0a9ca7b911625b0315ffddfcb8/A in TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:21,160 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23525a0a9ca7b911625b0315ffddfcb8/B in TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:21,161 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/42688fda3cd548159111b977f8b0ce55, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/cdaee554da47438b994bcc686641b0db, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/0f3d1b57feee459da2f9ffd260c2277d] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp, totalSize=35.2 K 2024-11-17T08:15:21,161 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/75b633cb4ce64f1faea17a12b70eccf7, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/6890b7afd8124947a2a8feba3307f071, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/2e781ec865144a6b893f9d42e0cd87b7] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp, totalSize=35.2 K 2024-11-17T08:15:21,161 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75b633cb4ce64f1faea17a12b70eccf7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731831318734 2024-11-17T08:15:21,161 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 42688fda3cd548159111b977f8b0ce55, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731831318734 2024-11-17T08:15:21,161 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting cdaee554da47438b994bcc686641b0db, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731831318814 2024-11-17T08:15:21,161 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6890b7afd8124947a2a8feba3307f071, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731831318814 2024-11-17T08:15:21,161 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f3d1b57feee459da2f9ffd260c2277d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731831319507 2024-11-17T08:15:21,161 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e781ec865144a6b893f9d42e0cd87b7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731831319507 2024-11-17T08:15:21,169 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23525a0a9ca7b911625b0315ffddfcb8#A#compaction#192 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:21,169 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/818e0eccbebd4b978e3584727e5aed1e is 50, key is test_row_0/A:col10/1731831319507/Put/seqid=0 2024-11-17T08:15:21,174 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23525a0a9ca7b911625b0315ffddfcb8#B#compaction#193 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:21,174 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/ca9043c695794e1481a66b8510898a43 is 50, key is test_row_0/B:col10/1731831319507/Put/seqid=0 2024-11-17T08:15:21,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-17T08:15:21,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:21,188 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 23525a0a9ca7b911625b0315ffddfcb8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-17T08:15:21,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=A 2024-11-17T08:15:21,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:21,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=B 2024-11-17T08:15:21,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:21,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=C 2024-11-17T08:15:21,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:21,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-17T08:15:21,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742059_1235 (size=12104) 2024-11-17T08:15:21,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742060_1236 (size=12104) 2024-11-17T08:15:21,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/0c7330ebde3c432099230a3932394504 is 50, key is test_row_0/A:col10/1731831320697/Put/seqid=0 2024-11-17T08:15:21,216 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/818e0eccbebd4b978e3584727e5aed1e as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/818e0eccbebd4b978e3584727e5aed1e 2024-11-17T08:15:21,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742061_1237 (size=12001) 2024-11-17T08:15:21,222 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23525a0a9ca7b911625b0315ffddfcb8/A of 23525a0a9ca7b911625b0315ffddfcb8 into 818e0eccbebd4b978e3584727e5aed1e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:21,222 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:21,222 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., storeName=23525a0a9ca7b911625b0315ffddfcb8/A, priority=13, startTime=1731831321159; duration=0sec 2024-11-17T08:15:21,222 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:15:21,222 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23525a0a9ca7b911625b0315ffddfcb8:A 2024-11-17T08:15:21,222 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:15:21,225 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:15:21,225 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): 23525a0a9ca7b911625b0315ffddfcb8/C is initiating minor compaction (all files) 2024-11-17T08:15:21,225 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23525a0a9ca7b911625b0315ffddfcb8/C in TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:21,225 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/0e06d9a598a745f0a7e175a008654198, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/78a3dbe261a34d94a7758e985f6bc1a7, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/59e5978a22824bdea6796897ddbce226] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp, totalSize=35.2 K 2024-11-17T08:15:21,226 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e06d9a598a745f0a7e175a008654198, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731831318734 2024-11-17T08:15:21,226 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78a3dbe261a34d94a7758e985f6bc1a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731831318814 2024-11-17T08:15:21,226 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59e5978a22824bdea6796897ddbce226, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731831319507 2024-11-17T08:15:21,246 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23525a0a9ca7b911625b0315ffddfcb8#C#compaction#195 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:21,247 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/a646f9f774df4b16a71bcba14a6746fe is 50, key is test_row_0/C:col10/1731831319507/Put/seqid=0 2024-11-17T08:15:21,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742062_1238 (size=12104) 2024-11-17T08:15:21,282 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/a646f9f774df4b16a71bcba14a6746fe as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/a646f9f774df4b16a71bcba14a6746fe 2024-11-17T08:15:21,289 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23525a0a9ca7b911625b0315ffddfcb8/C of 23525a0a9ca7b911625b0315ffddfcb8 into a646f9f774df4b16a71bcba14a6746fe(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:21,289 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:21,289 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., storeName=23525a0a9ca7b911625b0315ffddfcb8/C, priority=13, startTime=1731831321159; duration=0sec 2024-11-17T08:15:21,290 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:21,290 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23525a0a9ca7b911625b0315ffddfcb8:C 2024-11-17T08:15:21,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 23525a0a9ca7b911625b0315ffddfcb8 2024-11-17T08:15:21,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. as already flushing 2024-11-17T08:15:21,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:21,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831381342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:21,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:21,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:21,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831381344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:21,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:21,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831381344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:21,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:21,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:21,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:21,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831381346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:21,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:21,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:21,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831381454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:21,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:21,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831381454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:21,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:21,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:21,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831381454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:21,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:21,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:21,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:21,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831381454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:21,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:21,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-17T08:15:21,612 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/ca9043c695794e1481a66b8510898a43 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/ca9043c695794e1481a66b8510898a43 2024-11-17T08:15:21,619 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23525a0a9ca7b911625b0315ffddfcb8/B of 23525a0a9ca7b911625b0315ffddfcb8 into ca9043c695794e1481a66b8510898a43(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:21,619 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:21,619 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., storeName=23525a0a9ca7b911625b0315ffddfcb8/B, priority=13, startTime=1731831321159; duration=0sec 2024-11-17T08:15:21,619 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:21,619 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23525a0a9ca7b911625b0315ffddfcb8:B 2024-11-17T08:15:21,620 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/0c7330ebde3c432099230a3932394504 2024-11-17T08:15:21,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/414dad6927d5444e86873fa316906adf is 50, key is test_row_0/B:col10/1731831320697/Put/seqid=0 2024-11-17T08:15:21,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742063_1239 (size=12001) 2024-11-17T08:15:21,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:21,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831381664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:21,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:21,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831381664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:21,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:21,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:21,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:21,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831381665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:21,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:21,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:21,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831381665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:21,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:21,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:21,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831381973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:21,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:21,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831381973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:21,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:21,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:21,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:21,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831381974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831381974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:21,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:21,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:21,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:22,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-17T08:15:22,039 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/414dad6927d5444e86873fa316906adf 2024-11-17T08:15:22,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/315dfcbc42484589b10f694ed68a847e is 50, key is test_row_0/C:col10/1731831320697/Put/seqid=0 2024-11-17T08:15:22,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742064_1240 (size=12001) 2024-11-17T08:15:22,078 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/315dfcbc42484589b10f694ed68a847e 2024-11-17T08:15:22,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/0c7330ebde3c432099230a3932394504 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/0c7330ebde3c432099230a3932394504 2024-11-17T08:15:22,092 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/0c7330ebde3c432099230a3932394504, entries=150, sequenceid=80, filesize=11.7 K 2024-11-17T08:15:22,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/414dad6927d5444e86873fa316906adf as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/414dad6927d5444e86873fa316906adf 2024-11-17T08:15:22,108 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/414dad6927d5444e86873fa316906adf, entries=150, sequenceid=80, filesize=11.7 K 2024-11-17T08:15:22,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/315dfcbc42484589b10f694ed68a847e as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/315dfcbc42484589b10f694ed68a847e 2024-11-17T08:15:22,117 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/315dfcbc42484589b10f694ed68a847e, entries=150, sequenceid=80, filesize=11.7 K 2024-11-17T08:15:22,118 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 23525a0a9ca7b911625b0315ffddfcb8 in 930ms, sequenceid=80, compaction requested=false 2024-11-17T08:15:22,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:22,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:22,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-17T08:15:22,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-17T08:15:22,128 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-17T08:15:22,128 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2450 sec 2024-11-17T08:15:22,133 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.2560 sec 2024-11-17T08:15:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 23525a0a9ca7b911625b0315ffddfcb8 2024-11-17T08:15:22,487 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23525a0a9ca7b911625b0315ffddfcb8 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-17T08:15:22,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=A 2024-11-17T08:15:22,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:22,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=B 2024-11-17T08:15:22,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:22,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=C 2024-11-17T08:15:22,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:22,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/84001bdf570d4fcd8f71211646f87260 is 50, key is test_row_0/A:col10/1731831322486/Put/seqid=0 2024-11-17T08:15:22,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742065_1241 (size=16681) 2024-11-17T08:15:22,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/84001bdf570d4fcd8f71211646f87260 2024-11-17T08:15:22,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/781114373aa9499bbe8554b8f7c9d2ff is 50, key is test_row_0/B:col10/1731831322486/Put/seqid=0 2024-11-17T08:15:22,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742066_1242 (size=12001) 2024-11-17T08:15:22,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831382504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:22,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831382505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:22,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:22,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:22,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:22,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:22,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831382506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:22,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:22,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:22,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:22,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:22,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:22,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831382510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:22,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:22,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:22,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:22,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:22,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831382613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:22,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:22,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:22,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:22,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831382613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:22,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:22,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:22,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:22,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831382614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:22,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:22,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:22,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:22,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831382624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:22,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:22,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:22,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:22,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831382823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:22,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:22,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:22,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:22,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831382824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:22,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:22,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:22,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:22,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831382824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:22,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:22,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:22,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:22,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831382834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:22,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:22,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:22,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:22,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/781114373aa9499bbe8554b8f7c9d2ff 2024-11-17T08:15:22,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/e3b49af6f749459ebfbff1330a2f7ef6 is 50, key is test_row_0/C:col10/1731831322486/Put/seqid=0 2024-11-17T08:15:22,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742067_1243 (size=12001) 2024-11-17T08:15:23,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-17T08:15:23,023 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:15:23,024 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:15:23,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-17T08:15:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T08:15:23,026 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:15:23,027 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:15:23,027 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:15:23,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:23,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53876 deadline: 1731831383063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:23,068 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to e3d0adb87c01,40631,1731831247805 for 'test_row_2' in 23525a0a9ca7b911625b0315ffddfcb8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T08:15:23,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:23,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:23,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:23,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T08:15:23,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:23,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831383133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:23,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:23,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831383133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:23,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:23,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:23,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:23,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:23,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:23,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831383133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:23,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:23,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:23,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:23,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:23,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831383144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:23,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:23,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:23,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:23,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-17T08:15:23,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:23,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. as already flushing 2024-11-17T08:15:23,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:23,181 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:23,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:23,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:23,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/e3b49af6f749459ebfbff1330a2f7ef6 2024-11-17T08:15:23,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/84001bdf570d4fcd8f71211646f87260 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/84001bdf570d4fcd8f71211646f87260 2024-11-17T08:15:23,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/84001bdf570d4fcd8f71211646f87260, entries=250, sequenceid=99, filesize=16.3 K 2024-11-17T08:15:23,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/781114373aa9499bbe8554b8f7c9d2ff as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/781114373aa9499bbe8554b8f7c9d2ff 2024-11-17T08:15:23,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-17T08:15:23,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:23,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. as already flushing 2024-11-17T08:15:23,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:23,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:23,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:23,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:23,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/781114373aa9499bbe8554b8f7c9d2ff, entries=150, sequenceid=99, filesize=11.7 K 2024-11-17T08:15:23,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/e3b49af6f749459ebfbff1330a2f7ef6 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/e3b49af6f749459ebfbff1330a2f7ef6 2024-11-17T08:15:23,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/e3b49af6f749459ebfbff1330a2f7ef6, entries=150, sequenceid=99, filesize=11.7 K 2024-11-17T08:15:23,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T08:15:23,345 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 23525a0a9ca7b911625b0315ffddfcb8 in 858ms, sequenceid=99, compaction requested=true 2024-11-17T08:15:23,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:23,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23525a0a9ca7b911625b0315ffddfcb8:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:15:23,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:23,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23525a0a9ca7b911625b0315ffddfcb8:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:15:23,345 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:15:23,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:15:23,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23525a0a9ca7b911625b0315ffddfcb8:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:15:23,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T08:15:23,345 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:15:23,346 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:15:23,346 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:15:23,346 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): 23525a0a9ca7b911625b0315ffddfcb8/B is initiating minor compaction (all files) 2024-11-17T08:15:23,346 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): 23525a0a9ca7b911625b0315ffddfcb8/A is initiating minor compaction (all files) 2024-11-17T08:15:23,347 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23525a0a9ca7b911625b0315ffddfcb8/A in TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:23,347 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23525a0a9ca7b911625b0315ffddfcb8/B in TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:23,347 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/ca9043c695794e1481a66b8510898a43, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/414dad6927d5444e86873fa316906adf, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/781114373aa9499bbe8554b8f7c9d2ff] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp, totalSize=35.3 K 2024-11-17T08:15:23,347 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/818e0eccbebd4b978e3584727e5aed1e, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/0c7330ebde3c432099230a3932394504, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/84001bdf570d4fcd8f71211646f87260] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp, totalSize=39.8 K 2024-11-17T08:15:23,347 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 818e0eccbebd4b978e3584727e5aed1e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731831319507 2024-11-17T08:15:23,347 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting ca9043c695794e1481a66b8510898a43, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731831319507 2024-11-17T08:15:23,347 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 414dad6927d5444e86873fa316906adf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731831320682 2024-11-17T08:15:23,347 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c7330ebde3c432099230a3932394504, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731831320682 2024-11-17T08:15:23,348 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84001bdf570d4fcd8f71211646f87260, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731831321342 2024-11-17T08:15:23,348 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 781114373aa9499bbe8554b8f7c9d2ff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731831321342 2024-11-17T08:15:23,376 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23525a0a9ca7b911625b0315ffddfcb8#A#compaction#201 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:23,376 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/905c356fe62f44c8a4a9d80c967db163 is 50, key is test_row_0/A:col10/1731831322486/Put/seqid=0 2024-11-17T08:15:23,387 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23525a0a9ca7b911625b0315ffddfcb8#B#compaction#202 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:23,388 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/79e06c8ff51e4f5f850c94b9b620d138 is 50, key is test_row_0/B:col10/1731831322486/Put/seqid=0 2024-11-17T08:15:23,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742069_1245 (size=12207) 2024-11-17T08:15:23,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742068_1244 (size=12207) 2024-11-17T08:15:23,442 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/79e06c8ff51e4f5f850c94b9b620d138 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/79e06c8ff51e4f5f850c94b9b620d138 2024-11-17T08:15:23,445 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/905c356fe62f44c8a4a9d80c967db163 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/905c356fe62f44c8a4a9d80c967db163 2024-11-17T08:15:23,451 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23525a0a9ca7b911625b0315ffddfcb8/B of 23525a0a9ca7b911625b0315ffddfcb8 into 79e06c8ff51e4f5f850c94b9b620d138(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:23,451 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:23,451 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., storeName=23525a0a9ca7b911625b0315ffddfcb8/B, priority=13, startTime=1731831323345; duration=0sec 2024-11-17T08:15:23,451 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:15:23,451 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23525a0a9ca7b911625b0315ffddfcb8:B 2024-11-17T08:15:23,451 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:15:23,453 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:15:23,453 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): 23525a0a9ca7b911625b0315ffddfcb8/C is initiating minor compaction (all files) 2024-11-17T08:15:23,453 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23525a0a9ca7b911625b0315ffddfcb8/C in TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:23,453 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/a646f9f774df4b16a71bcba14a6746fe, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/315dfcbc42484589b10f694ed68a847e, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/e3b49af6f749459ebfbff1330a2f7ef6] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp, totalSize=35.3 K 2024-11-17T08:15:23,453 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting a646f9f774df4b16a71bcba14a6746fe, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731831319507 2024-11-17T08:15:23,454 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 315dfcbc42484589b10f694ed68a847e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731831320682 2024-11-17T08:15:23,454 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23525a0a9ca7b911625b0315ffddfcb8/A of 23525a0a9ca7b911625b0315ffddfcb8 into 905c356fe62f44c8a4a9d80c967db163(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:23,454 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:23,454 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., storeName=23525a0a9ca7b911625b0315ffddfcb8/A, priority=13, startTime=1731831323345; duration=0sec 2024-11-17T08:15:23,454 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:23,454 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting e3b49af6f749459ebfbff1330a2f7ef6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731831321342 2024-11-17T08:15:23,454 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23525a0a9ca7b911625b0315ffddfcb8:A 2024-11-17T08:15:23,465 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23525a0a9ca7b911625b0315ffddfcb8#C#compaction#203 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:23,465 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/9d00d13b4ed247d79dbeb208975e2a34 is 50, key is test_row_0/C:col10/1731831322486/Put/seqid=0 2024-11-17T08:15:23,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-17T08:15:23,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:23,489 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 23525a0a9ca7b911625b0315ffddfcb8 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-17T08:15:23,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=A 2024-11-17T08:15:23,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:23,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=B 2024-11-17T08:15:23,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:23,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=C 2024-11-17T08:15:23,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:23,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742070_1246 (size=12207) 2024-11-17T08:15:23,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/0e985e982d66493cb376e07c3cf301b8 is 50, key is test_row_0/A:col10/1731831322504/Put/seqid=0 2024-11-17T08:15:23,502 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/9d00d13b4ed247d79dbeb208975e2a34 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/9d00d13b4ed247d79dbeb208975e2a34 2024-11-17T08:15:23,510 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23525a0a9ca7b911625b0315ffddfcb8/C of 23525a0a9ca7b911625b0315ffddfcb8 into 9d00d13b4ed247d79dbeb208975e2a34(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:23,510 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:23,510 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., storeName=23525a0a9ca7b911625b0315ffddfcb8/C, priority=13, startTime=1731831323345; duration=0sec 2024-11-17T08:15:23,511 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:23,511 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23525a0a9ca7b911625b0315ffddfcb8:C 2024-11-17T08:15:23,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742071_1247 (size=12001) 2024-11-17T08:15:23,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 23525a0a9ca7b911625b0315ffddfcb8 2024-11-17T08:15:23,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. as already flushing 2024-11-17T08:15:23,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T08:15:23,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:23,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831383657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:23,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:23,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:23,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:23,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:23,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831383659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:23,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:23,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:23,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831383660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:23,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:23,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:23,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:23,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:23,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:23,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:23,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831383663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:23,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:23,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:23,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:23,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:23,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831383774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:23,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:23,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:23,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:23,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:23,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831383774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:23,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:23,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:23,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831383774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:23,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:23,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:23,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:23,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:23,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831383774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:23,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:23,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:23,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:23,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:23,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:23,935 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/0e985e982d66493cb376e07c3cf301b8 2024-11-17T08:15:23,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/5b873549d2a54c4482e9e124b18ad440 is 50, key is test_row_0/B:col10/1731831322504/Put/seqid=0 2024-11-17T08:15:23,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742072_1248 (size=12001) 2024-11-17T08:15:23,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:23,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831383984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:23,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:23,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:23,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:23,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831383985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:23,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:23,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:23,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:23,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:23,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:23,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831383985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:23,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:23,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831383985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:23,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:23,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:23,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:23,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:23,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:24,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T08:15:24,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:24,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831384293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:24,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:24,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:24,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:24,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831384293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:24,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:24,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:24,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:24,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:24,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:24,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831384293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:24,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:24,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:24,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:24,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:24,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831384294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:24,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:24,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:24,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:24,357 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/5b873549d2a54c4482e9e124b18ad440 2024-11-17T08:15:24,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/8f7645d8e62848db9b1d71a5da45e07c is 50, key is test_row_0/C:col10/1731831322504/Put/seqid=0 2024-11-17T08:15:24,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742073_1249 (size=12001) 2024-11-17T08:15:24,385 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/8f7645d8e62848db9b1d71a5da45e07c 2024-11-17T08:15:24,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/0e985e982d66493cb376e07c3cf301b8 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/0e985e982d66493cb376e07c3cf301b8 2024-11-17T08:15:24,397 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/0e985e982d66493cb376e07c3cf301b8, entries=150, sequenceid=117, filesize=11.7 K 2024-11-17T08:15:24,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/5b873549d2a54c4482e9e124b18ad440 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/5b873549d2a54c4482e9e124b18ad440 2024-11-17T08:15:24,403 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/5b873549d2a54c4482e9e124b18ad440, entries=150, sequenceid=117, filesize=11.7 K 2024-11-17T08:15:24,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/8f7645d8e62848db9b1d71a5da45e07c as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/8f7645d8e62848db9b1d71a5da45e07c 2024-11-17T08:15:24,414 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/8f7645d8e62848db9b1d71a5da45e07c, entries=150, sequenceid=117, filesize=11.7 K 2024-11-17T08:15:24,414 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 23525a0a9ca7b911625b0315ffddfcb8 in 926ms, sequenceid=117, compaction requested=false 2024-11-17T08:15:24,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:24,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:24,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-17T08:15:24,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-17T08:15:24,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-17T08:15:24,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3890 sec 2024-11-17T08:15:24,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.3980 sec 2024-11-17T08:15:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 23525a0a9ca7b911625b0315ffddfcb8 2024-11-17T08:15:24,807 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23525a0a9ca7b911625b0315ffddfcb8 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-17T08:15:24,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=A 2024-11-17T08:15:24,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:24,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=B 2024-11-17T08:15:24,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:24,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=C 2024-11-17T08:15:24,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:24,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/1e85ad80bf764da0ba214a4e15adf5f3 is 50, key is test_row_0/A:col10/1731831324806/Put/seqid=0 2024-11-17T08:15:24,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742074_1250 (size=12101) 2024-11-17T08:15:24,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/1e85ad80bf764da0ba214a4e15adf5f3 2024-11-17T08:15:24,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/b05e41b761c24f5097ce45a5bc696517 is 50, key is test_row_0/B:col10/1731831324806/Put/seqid=0 2024-11-17T08:15:24,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:24,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831384819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:24,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:24,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:24,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:24,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831384820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:24,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:24,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:24,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:24,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:24,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831384822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:24,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:24,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:24,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:24,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831384824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:24,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:24,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:24,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:24,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742075_1251 (size=12101) 2024-11-17T08:15:24,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:24,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831384933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:24,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:24,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:24,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:24,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831384933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:24,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:24,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:24,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:24,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:24,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:24,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831384933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:24,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:24,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:24,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:24,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:24,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831384933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:24,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:24,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:24,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:25,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:25,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:25,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831385144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:25,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831385144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:25,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:25,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:25,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831385144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:25,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831385144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:25,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:25,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:25,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:25,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:25,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:25,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:25,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:25,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:25,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:25,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:25,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:25,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:25,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T08:15:25,173 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:15:25,174 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:15:25,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-17T08:15:25,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T08:15:25,177 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:15:25,178 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:15:25,178 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:15:25,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/b05e41b761c24f5097ce45a5bc696517 2024-11-17T08:15:25,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/2bf5beb49fcb4743b94d41c0e0e827a8 is 50, key is test_row_0/C:col10/1731831324806/Put/seqid=0 2024-11-17T08:15:25,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742076_1252 (size=12101) 2024-11-17T08:15:25,264 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/2bf5beb49fcb4743b94d41c0e0e827a8 2024-11-17T08:15:25,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/1e85ad80bf764da0ba214a4e15adf5f3 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/1e85ad80bf764da0ba214a4e15adf5f3 2024-11-17T08:15:25,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/1e85ad80bf764da0ba214a4e15adf5f3, entries=150, sequenceid=135, filesize=11.8 K 2024-11-17T08:15:25,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/b05e41b761c24f5097ce45a5bc696517 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/b05e41b761c24f5097ce45a5bc696517 2024-11-17T08:15:25,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/b05e41b761c24f5097ce45a5bc696517, entries=150, sequenceid=135, filesize=11.8 K 2024-11-17T08:15:25,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/2bf5beb49fcb4743b94d41c0e0e827a8 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/2bf5beb49fcb4743b94d41c0e0e827a8 2024-11-17T08:15:25,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T08:15:25,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/2bf5beb49fcb4743b94d41c0e0e827a8, entries=150, sequenceid=135, filesize=11.8 K 2024-11-17T08:15:25,285 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 23525a0a9ca7b911625b0315ffddfcb8 in 478ms, sequenceid=135, compaction requested=true 2024-11-17T08:15:25,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:25,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23525a0a9ca7b911625b0315ffddfcb8:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:15:25,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:25,285 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:15:25,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23525a0a9ca7b911625b0315ffddfcb8:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:15:25,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:25,285 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:15:25,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23525a0a9ca7b911625b0315ffddfcb8:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:15:25,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:15:25,287 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:15:25,287 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): 23525a0a9ca7b911625b0315ffddfcb8/B is initiating minor compaction (all files) 2024-11-17T08:15:25,287 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23525a0a9ca7b911625b0315ffddfcb8/B in TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:25,287 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/79e06c8ff51e4f5f850c94b9b620d138, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/5b873549d2a54c4482e9e124b18ad440, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/b05e41b761c24f5097ce45a5bc696517] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp, totalSize=35.5 K 2024-11-17T08:15:25,287 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 79e06c8ff51e4f5f850c94b9b620d138, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731831321342 2024-11-17T08:15:25,287 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:15:25,288 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): 23525a0a9ca7b911625b0315ffddfcb8/A is initiating minor compaction (all files) 2024-11-17T08:15:25,288 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23525a0a9ca7b911625b0315ffddfcb8/A in TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:25,288 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/905c356fe62f44c8a4a9d80c967db163, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/0e985e982d66493cb376e07c3cf301b8, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/1e85ad80bf764da0ba214a4e15adf5f3] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp, totalSize=35.5 K 2024-11-17T08:15:25,289 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 905c356fe62f44c8a4a9d80c967db163, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731831321342 2024-11-17T08:15:25,289 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b873549d2a54c4482e9e124b18ad440, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731831322490 2024-11-17T08:15:25,289 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e985e982d66493cb376e07c3cf301b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731831322490 2024-11-17T08:15:25,289 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting b05e41b761c24f5097ce45a5bc696517, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731831323649 2024-11-17T08:15:25,290 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e85ad80bf764da0ba214a4e15adf5f3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731831323649 2024-11-17T08:15:25,301 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23525a0a9ca7b911625b0315ffddfcb8#B#compaction#210 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:25,301 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/3aa74db4167e463085e2f96da3d1bab9 is 50, key is test_row_0/B:col10/1731831324806/Put/seqid=0 2024-11-17T08:15:25,310 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23525a0a9ca7b911625b0315ffddfcb8#A#compaction#211 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:25,310 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/8486c769e35b4506ae6befabbb8827c2 is 50, key is test_row_0/A:col10/1731831324806/Put/seqid=0 2024-11-17T08:15:25,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-17T08:15:25,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:25,333 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 23525a0a9ca7b911625b0315ffddfcb8 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-17T08:15:25,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=A 2024-11-17T08:15:25,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:25,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=B 2024-11-17T08:15:25,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:25,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=C 2024-11-17T08:15:25,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:25,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742078_1254 (size=12409) 2024-11-17T08:15:25,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742077_1253 (size=12409) 2024-11-17T08:15:25,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/fbaeee0be0ba44ed92d1ee14a6c93394 is 50, key is test_row_0/A:col10/1731831324808/Put/seqid=0 2024-11-17T08:15:25,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742079_1255 (size=12151) 2024-11-17T08:15:25,363 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/fbaeee0be0ba44ed92d1ee14a6c93394 2024-11-17T08:15:25,364 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/3aa74db4167e463085e2f96da3d1bab9 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/3aa74db4167e463085e2f96da3d1bab9 2024-11-17T08:15:25,369 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23525a0a9ca7b911625b0315ffddfcb8/B of 23525a0a9ca7b911625b0315ffddfcb8 into 3aa74db4167e463085e2f96da3d1bab9(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:25,369 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:25,371 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., storeName=23525a0a9ca7b911625b0315ffddfcb8/B, priority=13, startTime=1731831325285; duration=0sec 2024-11-17T08:15:25,371 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:15:25,371 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23525a0a9ca7b911625b0315ffddfcb8:B 2024-11-17T08:15:25,371 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:15:25,372 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:15:25,372 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): 23525a0a9ca7b911625b0315ffddfcb8/C is initiating minor compaction (all files) 2024-11-17T08:15:25,372 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23525a0a9ca7b911625b0315ffddfcb8/C in TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:25,372 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/9d00d13b4ed247d79dbeb208975e2a34, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/8f7645d8e62848db9b1d71a5da45e07c, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/2bf5beb49fcb4743b94d41c0e0e827a8] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp, totalSize=35.5 K 2024-11-17T08:15:25,373 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d00d13b4ed247d79dbeb208975e2a34, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731831321342 2024-11-17T08:15:25,373 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f7645d8e62848db9b1d71a5da45e07c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731831322490 2024-11-17T08:15:25,373 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bf5beb49fcb4743b94d41c0e0e827a8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731831323649 2024-11-17T08:15:25,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/34ccb4809b7042b19b2b022b1a86fb56 is 50, key is test_row_0/B:col10/1731831324808/Put/seqid=0 2024-11-17T08:15:25,388 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23525a0a9ca7b911625b0315ffddfcb8#C#compaction#214 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:25,388 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/7d34eec3a9a8489391804b7c543c9324 is 50, key is test_row_0/C:col10/1731831324806/Put/seqid=0 2024-11-17T08:15:25,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742080_1256 (size=12151) 2024-11-17T08:15:25,394 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/34ccb4809b7042b19b2b022b1a86fb56 2024-11-17T08:15:25,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742081_1257 (size=12409) 2024-11-17T08:15:25,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/41cb4ad82f054c5baae27142e961264d is 50, key is test_row_0/C:col10/1731831324808/Put/seqid=0 2024-11-17T08:15:25,437 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/7d34eec3a9a8489391804b7c543c9324 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/7d34eec3a9a8489391804b7c543c9324 2024-11-17T08:15:25,447 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23525a0a9ca7b911625b0315ffddfcb8/C of 23525a0a9ca7b911625b0315ffddfcb8 into 7d34eec3a9a8489391804b7c543c9324(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:25,447 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:25,447 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., storeName=23525a0a9ca7b911625b0315ffddfcb8/C, priority=13, startTime=1731831325285; duration=0sec 2024-11-17T08:15:25,447 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:25,447 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23525a0a9ca7b911625b0315ffddfcb8:C 2024-11-17T08:15:25,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 23525a0a9ca7b911625b0315ffddfcb8 2024-11-17T08:15:25,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. as already flushing 2024-11-17T08:15:25,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742082_1258 (size=12151) 2024-11-17T08:15:25,469 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/41cb4ad82f054c5baae27142e961264d 2024-11-17T08:15:25,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/fbaeee0be0ba44ed92d1ee14a6c93394 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/fbaeee0be0ba44ed92d1ee14a6c93394 2024-11-17T08:15:25,480 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/fbaeee0be0ba44ed92d1ee14a6c93394, entries=150, sequenceid=150, filesize=11.9 K 2024-11-17T08:15:25,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/34ccb4809b7042b19b2b022b1a86fb56 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/34ccb4809b7042b19b2b022b1a86fb56 2024-11-17T08:15:25,485 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/34ccb4809b7042b19b2b022b1a86fb56, entries=150, sequenceid=150, filesize=11.9 K 2024-11-17T08:15:25,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:25,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:25,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831385476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:25,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831385479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:25,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:25,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:25,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:25,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:25,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:25,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:25,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:25,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831385479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:25,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:25,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:25,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:25,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:25,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831385480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:25,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:25,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:25,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:25,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/41cb4ad82f054c5baae27142e961264d as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/41cb4ad82f054c5baae27142e961264d 2024-11-17T08:15:25,491 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/41cb4ad82f054c5baae27142e961264d, entries=150, sequenceid=150, filesize=11.9 K 2024-11-17T08:15:25,492 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 23525a0a9ca7b911625b0315ffddfcb8 in 159ms, sequenceid=150, compaction requested=false 2024-11-17T08:15:25,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:25,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:25,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-17T08:15:25,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-17T08:15:25,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T08:15:25,496 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-17T08:15:25,496 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 316 msec 2024-11-17T08:15:25,499 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 323 msec 2024-11-17T08:15:25,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 23525a0a9ca7b911625b0315ffddfcb8 2024-11-17T08:15:25,597 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23525a0a9ca7b911625b0315ffddfcb8 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-17T08:15:25,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=A 2024-11-17T08:15:25,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:25,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=B 2024-11-17T08:15:25,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:25,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=C 2024-11-17T08:15:25,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:25,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/a4d0c85b8e2244d8b0f520895736d206 is 50, key is test_row_0/A:col10/1731831325473/Put/seqid=0 2024-11-17T08:15:25,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742083_1259 (size=12151) 2024-11-17T08:15:25,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/a4d0c85b8e2244d8b0f520895736d206 2024-11-17T08:15:25,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:25,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831385614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:25,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:25,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:25,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:25,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831385616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:25,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:25,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:25,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:25,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:25,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:25,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831385618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:25,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:25,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831385620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:25,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:25,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:25,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:25,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:25,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:25,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:25,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/d609d251a1dc4ec490d4f211fa4f3031 is 50, key is test_row_0/B:col10/1731831325473/Put/seqid=0 2024-11-17T08:15:25,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742084_1260 (size=12151) 2024-11-17T08:15:25,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:25,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831385733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:25,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:25,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:25,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:25,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:25,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831385734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:25,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:25,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:25,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:25,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:25,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831385734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:25,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:25,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:25,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:25,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831385734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:25,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:25,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:25,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:25,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:25,747 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/8486c769e35b4506ae6befabbb8827c2 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/8486c769e35b4506ae6befabbb8827c2 2024-11-17T08:15:25,752 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23525a0a9ca7b911625b0315ffddfcb8/A of 23525a0a9ca7b911625b0315ffddfcb8 into 8486c769e35b4506ae6befabbb8827c2(size=12.1 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:25,752 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:25,752 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., storeName=23525a0a9ca7b911625b0315ffddfcb8/A, priority=13, startTime=1731831325285; duration=0sec 2024-11-17T08:15:25,752 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:25,752 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23525a0a9ca7b911625b0315ffddfcb8:A 2024-11-17T08:15:25,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T08:15:25,804 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:15:25,804 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:15:25,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-17T08:15:25,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T08:15:25,806 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:15:25,806 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:15:25,806 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:15:25,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T08:15:25,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:25,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831385943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:25,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:25,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:25,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:25,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:25,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:25,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831385943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:25,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831385943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:25,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:25,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:25,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:25,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:25,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:25,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:25,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:25,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831385943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:25,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:25,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:25,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:25,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-17T08:15:25,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:25,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. as already flushing 2024-11-17T08:15:25,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:25,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:25,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:25,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:26,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/d609d251a1dc4ec490d4f211fa4f3031 2024-11-17T08:15:26,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/e662576b29cb48839081170c9aab3344 is 50, key is test_row_0/C:col10/1731831325473/Put/seqid=0 2024-11-17T08:15:26,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742085_1261 (size=12151) 2024-11-17T08:15:26,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/e662576b29cb48839081170c9aab3344 2024-11-17T08:15:26,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/a4d0c85b8e2244d8b0f520895736d206 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/a4d0c85b8e2244d8b0f520895736d206 2024-11-17T08:15:26,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/a4d0c85b8e2244d8b0f520895736d206, entries=150, sequenceid=170, filesize=11.9 K 2024-11-17T08:15:26,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/d609d251a1dc4ec490d4f211fa4f3031 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/d609d251a1dc4ec490d4f211fa4f3031 2024-11-17T08:15:26,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/d609d251a1dc4ec490d4f211fa4f3031, entries=150, sequenceid=170, filesize=11.9 K 2024-11-17T08:15:26,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/e662576b29cb48839081170c9aab3344 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/e662576b29cb48839081170c9aab3344 2024-11-17T08:15:26,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/e662576b29cb48839081170c9aab3344, entries=150, sequenceid=170, filesize=11.9 K 2024-11-17T08:15:26,093 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 23525a0a9ca7b911625b0315ffddfcb8 in 496ms, sequenceid=170, compaction requested=true 2024-11-17T08:15:26,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:26,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23525a0a9ca7b911625b0315ffddfcb8:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:15:26,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:26,093 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:15:26,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23525a0a9ca7b911625b0315ffddfcb8:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:15:26,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:26,093 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:15:26,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23525a0a9ca7b911625b0315ffddfcb8:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:15:26,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:15:26,094 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:15:26,094 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): 23525a0a9ca7b911625b0315ffddfcb8/B is initiating minor compaction (all files) 2024-11-17T08:15:26,095 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23525a0a9ca7b911625b0315ffddfcb8/B in TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:26,095 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/3aa74db4167e463085e2f96da3d1bab9, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/34ccb4809b7042b19b2b022b1a86fb56, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/d609d251a1dc4ec490d4f211fa4f3031] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp, totalSize=35.9 K 2024-11-17T08:15:26,095 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:15:26,095 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): 23525a0a9ca7b911625b0315ffddfcb8/A is initiating minor compaction (all files) 2024-11-17T08:15:26,095 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23525a0a9ca7b911625b0315ffddfcb8/A in TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:26,095 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/8486c769e35b4506ae6befabbb8827c2, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/fbaeee0be0ba44ed92d1ee14a6c93394, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/a4d0c85b8e2244d8b0f520895736d206] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp, totalSize=35.9 K 2024-11-17T08:15:26,095 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 3aa74db4167e463085e2f96da3d1bab9, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731831323649 2024-11-17T08:15:26,095 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8486c769e35b4506ae6befabbb8827c2, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731831323649 2024-11-17T08:15:26,096 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 34ccb4809b7042b19b2b022b1a86fb56, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731831324808 2024-11-17T08:15:26,096 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbaeee0be0ba44ed92d1ee14a6c93394, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731831324808 2024-11-17T08:15:26,097 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting d609d251a1dc4ec490d4f211fa4f3031, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731831325473 2024-11-17T08:15:26,097 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4d0c85b8e2244d8b0f520895736d206, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731831325473 2024-11-17T08:15:26,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-17T08:15:26,111 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23525a0a9ca7b911625b0315ffddfcb8#A#compaction#219 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:26,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:26,112 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 23525a0a9ca7b911625b0315ffddfcb8 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-17T08:15:26,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=A 2024-11-17T08:15:26,112 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/0e99033d738541baaa4249d1865e41b8 is 50, key is test_row_0/A:col10/1731831325473/Put/seqid=0 2024-11-17T08:15:26,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:26,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=B 2024-11-17T08:15:26,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:26,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=C 2024-11-17T08:15:26,112 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23525a0a9ca7b911625b0315ffddfcb8#B#compaction#220 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:26,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:26,112 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/d3b0ef541e24414facf46758adceb9b8 is 50, key is test_row_0/B:col10/1731831325473/Put/seqid=0 2024-11-17T08:15:26,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T08:15:26,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/579ac7af3e4f4d10834963770c34d21d is 50, key is test_row_0/A:col10/1731831325618/Put/seqid=0 2024-11-17T08:15:26,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742086_1262 (size=12561) 2024-11-17T08:15:26,154 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/0e99033d738541baaa4249d1865e41b8 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/0e99033d738541baaa4249d1865e41b8 2024-11-17T08:15:26,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742088_1264 (size=12151) 2024-11-17T08:15:26,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742087_1263 (size=12561) 2024-11-17T08:15:26,161 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23525a0a9ca7b911625b0315ffddfcb8/A of 23525a0a9ca7b911625b0315ffddfcb8 into 0e99033d738541baaa4249d1865e41b8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:26,162 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:26,162 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., storeName=23525a0a9ca7b911625b0315ffddfcb8/A, priority=13, startTime=1731831326093; duration=0sec 2024-11-17T08:15:26,162 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:15:26,162 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23525a0a9ca7b911625b0315ffddfcb8:A 2024-11-17T08:15:26,162 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:15:26,163 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:15:26,163 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): 23525a0a9ca7b911625b0315ffddfcb8/C is initiating minor compaction (all files) 2024-11-17T08:15:26,163 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23525a0a9ca7b911625b0315ffddfcb8/C in TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:26,164 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/7d34eec3a9a8489391804b7c543c9324, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/41cb4ad82f054c5baae27142e961264d, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/e662576b29cb48839081170c9aab3344] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp, totalSize=35.9 K 2024-11-17T08:15:26,164 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d34eec3a9a8489391804b7c543c9324, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731831323649 2024-11-17T08:15:26,164 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41cb4ad82f054c5baae27142e961264d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731831324808 2024-11-17T08:15:26,165 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting e662576b29cb48839081170c9aab3344, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731831325473 2024-11-17T08:15:26,207 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23525a0a9ca7b911625b0315ffddfcb8#C#compaction#222 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:26,207 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/e52673007b7b4c01a09801081f08c76c is 50, key is test_row_0/C:col10/1731831325473/Put/seqid=0 2024-11-17T08:15:26,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742089_1265 (size=12561) 2024-11-17T08:15:26,231 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/e52673007b7b4c01a09801081f08c76c as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/e52673007b7b4c01a09801081f08c76c 2024-11-17T08:15:26,241 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23525a0a9ca7b911625b0315ffddfcb8/C of 23525a0a9ca7b911625b0315ffddfcb8 into e52673007b7b4c01a09801081f08c76c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:26,241 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:26,241 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., storeName=23525a0a9ca7b911625b0315ffddfcb8/C, priority=13, startTime=1731831326093; duration=0sec 2024-11-17T08:15:26,241 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:26,241 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23525a0a9ca7b911625b0315ffddfcb8:C 2024-11-17T08:15:26,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. as already flushing 2024-11-17T08:15:26,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 23525a0a9ca7b911625b0315ffddfcb8 2024-11-17T08:15:26,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:26,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831386271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:26,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:26,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:26,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831386272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:26,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:26,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:26,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:26,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831386274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:26,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:26,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:26,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:26,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:26,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:26,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:26,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831386278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:26,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:26,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:26,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:26,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831386383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:26,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:26,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:26,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:26,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:26,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831386384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:26,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:26,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:26,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:26,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:26,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831386384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:26,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:26,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:26,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:26,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:26,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831386384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:26,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:26,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:26,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T08:15:26,557 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/579ac7af3e4f4d10834963770c34d21d 2024-11-17T08:15:26,563 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/d3b0ef541e24414facf46758adceb9b8 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/d3b0ef541e24414facf46758adceb9b8 2024-11-17T08:15:26,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/4b06111c7bf74ceebbad6b51bfb9078b is 50, key is test_row_0/B:col10/1731831325618/Put/seqid=0 2024-11-17T08:15:26,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742090_1266 (size=12151) 2024-11-17T08:15:26,570 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/4b06111c7bf74ceebbad6b51bfb9078b 2024-11-17T08:15:26,571 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23525a0a9ca7b911625b0315ffddfcb8/B of 23525a0a9ca7b911625b0315ffddfcb8 into d3b0ef541e24414facf46758adceb9b8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:26,571 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:26,571 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., storeName=23525a0a9ca7b911625b0315ffddfcb8/B, priority=13, startTime=1731831326093; duration=0sec 2024-11-17T08:15:26,571 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:26,571 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23525a0a9ca7b911625b0315ffddfcb8:B 2024-11-17T08:15:26,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/f15bc8b4c41c4d17ba3fdd8c41620303 is 50, key is test_row_0/C:col10/1731831325618/Put/seqid=0 2024-11-17T08:15:26,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742091_1267 (size=12151) 2024-11-17T08:15:26,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:26,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831386593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:26,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:26,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831386593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:26,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:26,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:26,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:26,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:26,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831386594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:26,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:26,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:26,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:26,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:26,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831386594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:26,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:26,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:26,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:26,601 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/f15bc8b4c41c4d17ba3fdd8c41620303 2024-11-17T08:15:26,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/579ac7af3e4f4d10834963770c34d21d as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/579ac7af3e4f4d10834963770c34d21d 2024-11-17T08:15:26,613 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/579ac7af3e4f4d10834963770c34d21d, entries=150, sequenceid=185, filesize=11.9 K 2024-11-17T08:15:26,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/4b06111c7bf74ceebbad6b51bfb9078b as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/4b06111c7bf74ceebbad6b51bfb9078b 2024-11-17T08:15:26,620 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/4b06111c7bf74ceebbad6b51bfb9078b, entries=150, sequenceid=185, filesize=11.9 K 2024-11-17T08:15:26,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/f15bc8b4c41c4d17ba3fdd8c41620303 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/f15bc8b4c41c4d17ba3fdd8c41620303 2024-11-17T08:15:26,627 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/f15bc8b4c41c4d17ba3fdd8c41620303, entries=150, sequenceid=185, filesize=11.9 K 2024-11-17T08:15:26,628 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 23525a0a9ca7b911625b0315ffddfcb8 in 517ms, sequenceid=185, compaction requested=false 2024-11-17T08:15:26,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:26,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:26,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-17T08:15:26,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-17T08:15:26,632 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-17T08:15:26,633 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 824 msec 2024-11-17T08:15:26,634 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 829 msec 2024-11-17T08:15:26,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 23525a0a9ca7b911625b0315ffddfcb8 2024-11-17T08:15:26,906 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23525a0a9ca7b911625b0315ffddfcb8 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-17T08:15:26,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=A 2024-11-17T08:15:26,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:26,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=B 2024-11-17T08:15:26,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:26,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=C 2024-11-17T08:15:26,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:26,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/8690451de16c4d1c95790bd167ffda20 is 50, key is test_row_0/A:col10/1731831326276/Put/seqid=0 2024-11-17T08:15:26,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742092_1268 (size=14541) 2024-11-17T08:15:26,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831386922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:26,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:26,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:26,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:26,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:26,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:26,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831386922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:26,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831386923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:26,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:26,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:26,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:26,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:26,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:26,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831386924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:26,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:26,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:26,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:26,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:26,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T08:15:26,944 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:15:26,945 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:15:26,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-17T08:15:26,946 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:15:26,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T08:15:26,947 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:15:26,947 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:15:27,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:27,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831387034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:27,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:27,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:27,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:27,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:27,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831387034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:27,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:27,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:27,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:27,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:27,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831387034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:27,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:27,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:27,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831387034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:27,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:27,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:27,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:27,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:27,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:27,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T08:15:27,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53876 deadline: 1731831387084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:27,088 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to e3d0adb87c01,40631,1731831247805 for 'test_row_2' in 23525a0a9ca7b911625b0315ffddfcb8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T08:15:27,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:27,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:27,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:27,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T08:15:27,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:27,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. as already flushing 2024-11-17T08:15:27,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:27,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:27,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:27,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:27,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831387243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:27,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:27,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831387243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:27,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:27,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:27,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:27,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:27,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:27,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:27,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831387244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:27,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:27,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:27,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:27,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:27,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:27,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831387244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:27,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:27,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:27,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:27,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T08:15:27,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:27,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. as already flushing 2024-11-17T08:15:27,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:27,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:27,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:27,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:27,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T08:15:27,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/8690451de16c4d1c95790bd167ffda20 2024-11-17T08:15:27,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/fabc637b5dff4b2aaa53686b79f021b4 is 50, key is test_row_0/B:col10/1731831326276/Put/seqid=0 2024-11-17T08:15:27,327 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-17T08:15:27,327 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-17T08:15:27,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742093_1269 (size=12151) 2024-11-17T08:15:27,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T08:15:27,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:27,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. as already flushing 2024-11-17T08:15:27,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:27,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:27,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:27,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:27,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:27,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831387553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:27,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:27,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:27,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:27,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:27,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:27,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831387554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:27,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:27,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831387554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:27,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:27,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:27,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:27,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831387553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:27,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:27,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:27,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:27,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:27,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:27,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:27,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T08:15:27,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:27,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. as already flushing 2024-11-17T08:15:27,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:27,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:27,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:27,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:27,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T08:15:27,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T08:15:27,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:27,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. as already flushing 2024-11-17T08:15:27,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:27,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:27,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:27,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:27,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/fabc637b5dff4b2aaa53686b79f021b4 2024-11-17T08:15:27,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/b472e4d7f94c4a8dbd39a18cd1f566ad is 50, key is test_row_0/C:col10/1731831326276/Put/seqid=0 2024-11-17T08:15:27,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742094_1270 (size=12151) 2024-11-17T08:15:27,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T08:15:27,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:27,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. as already flushing 2024-11-17T08:15:27,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:27,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:27,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:27,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:28,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T08:15:28,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:28,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. as already flushing 2024-11-17T08:15:28,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:28,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:28,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:28,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:28,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831388064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:28,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:28,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831388064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:28,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:28,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:28,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:28,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:28,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831388064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:28,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:28,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:28,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:28,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:28,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:28,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831388076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:28,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:28,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:28,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:28,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T08:15:28,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/b472e4d7f94c4a8dbd39a18cd1f566ad 2024-11-17T08:15:28,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/8690451de16c4d1c95790bd167ffda20 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/8690451de16c4d1c95790bd167ffda20 2024-11-17T08:15:28,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/8690451de16c4d1c95790bd167ffda20, entries=200, sequenceid=207, filesize=14.2 K 2024-11-17T08:15:28,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/fabc637b5dff4b2aaa53686b79f021b4 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/fabc637b5dff4b2aaa53686b79f021b4 2024-11-17T08:15:28,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/fabc637b5dff4b2aaa53686b79f021b4, entries=150, sequenceid=207, filesize=11.9 K 2024-11-17T08:15:28,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/b472e4d7f94c4a8dbd39a18cd1f566ad as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/b472e4d7f94c4a8dbd39a18cd1f566ad 2024-11-17T08:15:28,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/b472e4d7f94c4a8dbd39a18cd1f566ad, entries=150, sequenceid=207, filesize=11.9 K 2024-11-17T08:15:28,167 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 23525a0a9ca7b911625b0315ffddfcb8 in 1261ms, sequenceid=207, compaction requested=true 2024-11-17T08:15:28,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:28,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23525a0a9ca7b911625b0315ffddfcb8:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:15:28,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:28,167 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:15:28,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23525a0a9ca7b911625b0315ffddfcb8:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:15:28,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:28,167 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:15:28,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23525a0a9ca7b911625b0315ffddfcb8:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:15:28,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:15:28,169 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:15:28,169 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): 23525a0a9ca7b911625b0315ffddfcb8/A is initiating minor compaction (all files) 2024-11-17T08:15:28,169 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23525a0a9ca7b911625b0315ffddfcb8/A in TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:28,169 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/0e99033d738541baaa4249d1865e41b8, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/579ac7af3e4f4d10834963770c34d21d, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/8690451de16c4d1c95790bd167ffda20] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp, totalSize=38.3 K 2024-11-17T08:15:28,170 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:15:28,170 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e99033d738541baaa4249d1865e41b8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731831325473 2024-11-17T08:15:28,170 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): 23525a0a9ca7b911625b0315ffddfcb8/B is initiating minor compaction (all files) 2024-11-17T08:15:28,170 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23525a0a9ca7b911625b0315ffddfcb8/B in TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:28,170 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/d3b0ef541e24414facf46758adceb9b8, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/4b06111c7bf74ceebbad6b51bfb9078b, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/fabc637b5dff4b2aaa53686b79f021b4] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp, totalSize=36.0 K 2024-11-17T08:15:28,170 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 579ac7af3e4f4d10834963770c34d21d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731831325598 2024-11-17T08:15:28,170 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting d3b0ef541e24414facf46758adceb9b8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731831325473 2024-11-17T08:15:28,170 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8690451de16c4d1c95790bd167ffda20, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731831326266 2024-11-17T08:15:28,171 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b06111c7bf74ceebbad6b51bfb9078b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731831325598 2024-11-17T08:15:28,171 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting fabc637b5dff4b2aaa53686b79f021b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731831326268 2024-11-17T08:15:28,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T08:15:28,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:28,173 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 23525a0a9ca7b911625b0315ffddfcb8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-17T08:15:28,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=A 2024-11-17T08:15:28,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:28,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=B 2024-11-17T08:15:28,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:28,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=C 2024-11-17T08:15:28,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:28,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/dff2d328b3c647d1a4befe994e205807 is 50, key is test_row_0/A:col10/1731831326920/Put/seqid=0 2024-11-17T08:15:28,186 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23525a0a9ca7b911625b0315ffddfcb8#B#compaction#229 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:28,187 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/8c92f2557a844838bc90c99258faa77b is 50, key is test_row_0/B:col10/1731831326276/Put/seqid=0 2024-11-17T08:15:28,196 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23525a0a9ca7b911625b0315ffddfcb8#A#compaction#230 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:28,197 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/aec757bb230d4859a08ba36ee671eddc is 50, key is test_row_0/A:col10/1731831326276/Put/seqid=0 2024-11-17T08:15:28,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742095_1271 (size=12151) 2024-11-17T08:15:28,220 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/dff2d328b3c647d1a4befe994e205807 2024-11-17T08:15:28,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742097_1273 (size=12663) 2024-11-17T08:15:28,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742096_1272 (size=12663) 2024-11-17T08:15:28,243 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/aec757bb230d4859a08ba36ee671eddc as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/aec757bb230d4859a08ba36ee671eddc 2024-11-17T08:15:28,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/b50cc2d79234456c8640b3db69b4a699 is 50, key is test_row_0/B:col10/1731831326920/Put/seqid=0 2024-11-17T08:15:28,253 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/8c92f2557a844838bc90c99258faa77b as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/8c92f2557a844838bc90c99258faa77b 2024-11-17T08:15:28,255 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23525a0a9ca7b911625b0315ffddfcb8/A of 23525a0a9ca7b911625b0315ffddfcb8 into aec757bb230d4859a08ba36ee671eddc(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:28,255 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:28,255 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., storeName=23525a0a9ca7b911625b0315ffddfcb8/A, priority=13, startTime=1731831328167; duration=0sec 2024-11-17T08:15:28,256 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:15:28,256 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23525a0a9ca7b911625b0315ffddfcb8:A 2024-11-17T08:15:28,256 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:15:28,256 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:15:28,257 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): 23525a0a9ca7b911625b0315ffddfcb8/C is initiating minor compaction (all files) 2024-11-17T08:15:28,257 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23525a0a9ca7b911625b0315ffddfcb8/C in TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:28,257 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/e52673007b7b4c01a09801081f08c76c, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/f15bc8b4c41c4d17ba3fdd8c41620303, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/b472e4d7f94c4a8dbd39a18cd1f566ad] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp, totalSize=36.0 K 2024-11-17T08:15:28,257 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting e52673007b7b4c01a09801081f08c76c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731831325473 2024-11-17T08:15:28,259 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting f15bc8b4c41c4d17ba3fdd8c41620303, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731831325598 2024-11-17T08:15:28,259 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23525a0a9ca7b911625b0315ffddfcb8/B of 23525a0a9ca7b911625b0315ffddfcb8 into 8c92f2557a844838bc90c99258faa77b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:28,259 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:28,259 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., storeName=23525a0a9ca7b911625b0315ffddfcb8/B, priority=13, startTime=1731831328167; duration=0sec 2024-11-17T08:15:28,260 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting b472e4d7f94c4a8dbd39a18cd1f566ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731831326268 2024-11-17T08:15:28,260 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:28,260 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23525a0a9ca7b911625b0315ffddfcb8:B 2024-11-17T08:15:28,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742098_1274 (size=12151) 2024-11-17T08:15:28,274 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23525a0a9ca7b911625b0315ffddfcb8#C#compaction#232 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:28,275 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/a15d46d83b19472e857804275564b71a is 50, key is test_row_0/C:col10/1731831326276/Put/seqid=0 2024-11-17T08:15:28,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742099_1275 (size=12663) 2024-11-17T08:15:28,665 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/b50cc2d79234456c8640b3db69b4a699 2024-11-17T08:15:28,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/d4f897ccf8264f43aa6e13fc47f5bf45 is 50, key is test_row_0/C:col10/1731831326920/Put/seqid=0 2024-11-17T08:15:28,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742100_1276 (size=12151) 2024-11-17T08:15:28,688 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/d4f897ccf8264f43aa6e13fc47f5bf45 2024-11-17T08:15:28,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/dff2d328b3c647d1a4befe994e205807 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/dff2d328b3c647d1a4befe994e205807 2024-11-17T08:15:28,697 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/a15d46d83b19472e857804275564b71a as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/a15d46d83b19472e857804275564b71a 2024-11-17T08:15:28,709 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/dff2d328b3c647d1a4befe994e205807, entries=150, sequenceid=220, filesize=11.9 K 2024-11-17T08:15:28,712 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23525a0a9ca7b911625b0315ffddfcb8/C of 23525a0a9ca7b911625b0315ffddfcb8 into a15d46d83b19472e857804275564b71a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:28,713 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:28,713 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., storeName=23525a0a9ca7b911625b0315ffddfcb8/C, priority=13, startTime=1731831328167; duration=0sec 2024-11-17T08:15:28,713 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:28,713 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23525a0a9ca7b911625b0315ffddfcb8:C 2024-11-17T08:15:28,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/b50cc2d79234456c8640b3db69b4a699 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/b50cc2d79234456c8640b3db69b4a699 2024-11-17T08:15:28,717 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/b50cc2d79234456c8640b3db69b4a699, entries=150, sequenceid=220, filesize=11.9 K 2024-11-17T08:15:28,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/d4f897ccf8264f43aa6e13fc47f5bf45 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/d4f897ccf8264f43aa6e13fc47f5bf45 2024-11-17T08:15:28,729 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/d4f897ccf8264f43aa6e13fc47f5bf45, entries=150, sequenceid=220, filesize=11.9 K 2024-11-17T08:15:28,730 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for 23525a0a9ca7b911625b0315ffddfcb8 in 557ms, sequenceid=220, compaction requested=false 2024-11-17T08:15:28,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:28,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:28,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-17T08:15:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-17T08:15:28,734 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-17T08:15:28,734 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7850 sec 2024-11-17T08:15:28,738 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.7910 sec 2024-11-17T08:15:29,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 23525a0a9ca7b911625b0315ffddfcb8 2024-11-17T08:15:29,085 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23525a0a9ca7b911625b0315ffddfcb8 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-17T08:15:29,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=A 2024-11-17T08:15:29,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:29,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=B 2024-11-17T08:15:29,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:29,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=C 2024-11-17T08:15:29,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:29,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/4569d78750764c1fa49b2e61a97399ce is 50, key is test_row_0/A:col10/1731831329085/Put/seqid=0 2024-11-17T08:15:29,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T08:15:29,094 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:15:29,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742101_1277 (size=14541) 2024-11-17T08:15:29,095 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:15:29,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/4569d78750764c1fa49b2e61a97399ce 2024-11-17T08:15:29,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-17T08:15:29,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T08:15:29,097 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:15:29,098 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:15:29,098 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:15:29,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/8cfcc69e72d9408db28e5d629c9aa1d4 is 50, key is test_row_0/B:col10/1731831329085/Put/seqid=0 2024-11-17T08:15:29,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742102_1278 (size=12151) 2024-11-17T08:15:29,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:29,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831389109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:29,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:29,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:29,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:29,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831389110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:29,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:29,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:29,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:29,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831389111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:29,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:29,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:29,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:29,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831389112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:29,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:29,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:29,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T08:15:29,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:29,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831389223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:29,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:29,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:29,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:29,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831389224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:29,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:29,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:29,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:29,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831389224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:29,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:29,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:29,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831389224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:29,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:29,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:29,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:29,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:29,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-17T08:15:29,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:29,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. as already flushing 2024-11-17T08:15:29,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:29,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:29,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:29,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:29,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-17T08:15:29,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:29,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. as already flushing 2024-11-17T08:15:29,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:29,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:29,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:29,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:29,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T08:15:29,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:29,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831389434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:29,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:29,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:29,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831389434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:29,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:29,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:29,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:29,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:29,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:29,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831389434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:29,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:29,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:29,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:29,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:29,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831389434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:29,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:29,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:29,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:29,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/8cfcc69e72d9408db28e5d629c9aa1d4 2024-11-17T08:15:29,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/aba476b22aa4484d9684c784602181a7 is 50, key is test_row_0/C:col10/1731831329085/Put/seqid=0 2024-11-17T08:15:29,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742103_1279 (size=12151) 2024-11-17T08:15:29,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-17T08:15:29,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:29,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. as already flushing 2024-11-17T08:15:29,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:29,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:29,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:29,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:29,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-17T08:15:29,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:29,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. as already flushing 2024-11-17T08:15:29,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:29,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:29,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:29,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:29,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T08:15:29,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:29,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831389744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:29,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:29,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831389744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:29,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:29,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831389744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:29,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:29,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:29,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831389745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:29,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:29,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:29,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:29,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:29,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:29,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:29,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:29,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:29,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:29,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:29,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:29,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-17T08:15:29,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:29,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. as already flushing 2024-11-17T08:15:29,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:29,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:29,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:29,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:29,929 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/aba476b22aa4484d9684c784602181a7 2024-11-17T08:15:29,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/4569d78750764c1fa49b2e61a97399ce as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/4569d78750764c1fa49b2e61a97399ce 2024-11-17T08:15:29,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/4569d78750764c1fa49b2e61a97399ce, entries=200, sequenceid=235, filesize=14.2 K 2024-11-17T08:15:29,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/8cfcc69e72d9408db28e5d629c9aa1d4 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/8cfcc69e72d9408db28e5d629c9aa1d4 2024-11-17T08:15:29,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/8cfcc69e72d9408db28e5d629c9aa1d4, entries=150, sequenceid=235, filesize=11.9 K 2024-11-17T08:15:29,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/aba476b22aa4484d9684c784602181a7 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/aba476b22aa4484d9684c784602181a7 2024-11-17T08:15:29,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/aba476b22aa4484d9684c784602181a7, entries=150, sequenceid=235, filesize=11.9 K 2024-11-17T08:15:29,946 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 23525a0a9ca7b911625b0315ffddfcb8 in 861ms, sequenceid=235, compaction requested=true 2024-11-17T08:15:29,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:29,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23525a0a9ca7b911625b0315ffddfcb8:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:15:29,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:29,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23525a0a9ca7b911625b0315ffddfcb8:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:15:29,947 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:15:29,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:29,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23525a0a9ca7b911625b0315ffddfcb8:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:15:29,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:15:29,947 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:15:29,947 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39355 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:15:29,947 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:15:29,947 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): 23525a0a9ca7b911625b0315ffddfcb8/B is initiating minor compaction (all files) 2024-11-17T08:15:29,947 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): 23525a0a9ca7b911625b0315ffddfcb8/A is initiating minor compaction (all files) 2024-11-17T08:15:29,948 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23525a0a9ca7b911625b0315ffddfcb8/A in TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:29,948 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23525a0a9ca7b911625b0315ffddfcb8/B in TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:29,948 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/8c92f2557a844838bc90c99258faa77b, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/b50cc2d79234456c8640b3db69b4a699, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/8cfcc69e72d9408db28e5d629c9aa1d4] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp, totalSize=36.1 K 2024-11-17T08:15:29,948 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/aec757bb230d4859a08ba36ee671eddc, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/dff2d328b3c647d1a4befe994e205807, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/4569d78750764c1fa49b2e61a97399ce] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp, totalSize=38.4 K 2024-11-17T08:15:29,948 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c92f2557a844838bc90c99258faa77b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731831326268 2024-11-17T08:15:29,948 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting aec757bb230d4859a08ba36ee671eddc, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731831326268 2024-11-17T08:15:29,949 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting b50cc2d79234456c8640b3db69b4a699, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731831326920 2024-11-17T08:15:29,949 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting dff2d328b3c647d1a4befe994e205807, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731831326920 2024-11-17T08:15:29,950 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cfcc69e72d9408db28e5d629c9aa1d4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731831329084 2024-11-17T08:15:29,950 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4569d78750764c1fa49b2e61a97399ce, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731831329079 2024-11-17T08:15:29,956 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23525a0a9ca7b911625b0315ffddfcb8#B#compaction#237 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:29,956 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23525a0a9ca7b911625b0315ffddfcb8#A#compaction#238 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:29,957 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/a5c3ec4af12e4e19a7274b68b883f390 is 50, key is test_row_0/A:col10/1731831329085/Put/seqid=0 2024-11-17T08:15:29,957 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/13e8f2586e194b35956507bb09be41e3 is 50, key is test_row_0/B:col10/1731831329085/Put/seqid=0 2024-11-17T08:15:29,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742104_1280 (size=12765) 2024-11-17T08:15:29,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742105_1281 (size=12765) 2024-11-17T08:15:29,978 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/a5c3ec4af12e4e19a7274b68b883f390 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/a5c3ec4af12e4e19a7274b68b883f390 2024-11-17T08:15:29,983 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23525a0a9ca7b911625b0315ffddfcb8/A of 23525a0a9ca7b911625b0315ffddfcb8 into a5c3ec4af12e4e19a7274b68b883f390(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:29,983 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:29,983 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., storeName=23525a0a9ca7b911625b0315ffddfcb8/A, priority=13, startTime=1731831329946; duration=0sec 2024-11-17T08:15:29,983 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:15:29,983 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23525a0a9ca7b911625b0315ffddfcb8:A 2024-11-17T08:15:29,983 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:15:29,985 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:15:29,985 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): 23525a0a9ca7b911625b0315ffddfcb8/C is initiating minor compaction (all files) 2024-11-17T08:15:29,985 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23525a0a9ca7b911625b0315ffddfcb8/C in TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:29,985 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/a15d46d83b19472e857804275564b71a, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/d4f897ccf8264f43aa6e13fc47f5bf45, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/aba476b22aa4484d9684c784602181a7] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp, totalSize=36.1 K 2024-11-17T08:15:29,985 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting a15d46d83b19472e857804275564b71a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731831326268 2024-11-17T08:15:29,985 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4f897ccf8264f43aa6e13fc47f5bf45, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731831326920 2024-11-17T08:15:29,986 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting aba476b22aa4484d9684c784602181a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731831329084 2024-11-17T08:15:29,994 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23525a0a9ca7b911625b0315ffddfcb8#C#compaction#239 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:29,994 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/65c79b1932634d53a4c40c528cf842ef is 50, key is test_row_0/C:col10/1731831329085/Put/seqid=0 2024-11-17T08:15:30,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742106_1282 (size=12765) 2024-11-17T08:15:30,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-17T08:15:30,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:30,016 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 23525a0a9ca7b911625b0315ffddfcb8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-17T08:15:30,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=A 2024-11-17T08:15:30,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:30,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=B 2024-11-17T08:15:30,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:30,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=C 2024-11-17T08:15:30,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:30,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/1f621d86623044388aa9a6e4b1f135d1 is 50, key is test_row_0/A:col10/1731831329108/Put/seqid=0 2024-11-17T08:15:30,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742107_1283 (size=12151) 2024-11-17T08:15:30,031 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/1f621d86623044388aa9a6e4b1f135d1 2024-11-17T08:15:30,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/257df833c99c448fbbf7d3e7a298afd3 is 50, key is test_row_0/B:col10/1731831329108/Put/seqid=0 2024-11-17T08:15:30,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742108_1284 (size=12151) 2024-11-17T08:15:30,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T08:15:30,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 23525a0a9ca7b911625b0315ffddfcb8 2024-11-17T08:15:30,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. as already flushing 2024-11-17T08:15:30,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:30,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831390262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:30,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:30,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:30,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:30,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831390263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:30,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:30,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:30,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:30,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:30,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:30,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831390264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:30,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:30,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:30,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:30,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:30,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831390268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:30,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:30,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:30,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:30,376 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/13e8f2586e194b35956507bb09be41e3 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/13e8f2586e194b35956507bb09be41e3 2024-11-17T08:15:30,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:30,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831390374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:30,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:30,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:30,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:30,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:30,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831390374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:30,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:30,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:30,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:30,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831390374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:30,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:30,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:30,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:30,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:30,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:30,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831390374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:30,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:30,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:30,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:30,381 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23525a0a9ca7b911625b0315ffddfcb8/B of 23525a0a9ca7b911625b0315ffddfcb8 into 13e8f2586e194b35956507bb09be41e3(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:30,381 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:30,381 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., storeName=23525a0a9ca7b911625b0315ffddfcb8/B, priority=13, startTime=1731831329947; duration=0sec 2024-11-17T08:15:30,381 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:30,381 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23525a0a9ca7b911625b0315ffddfcb8:B 2024-11-17T08:15:30,408 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/65c79b1932634d53a4c40c528cf842ef as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/65c79b1932634d53a4c40c528cf842ef 2024-11-17T08:15:30,412 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23525a0a9ca7b911625b0315ffddfcb8/C of 23525a0a9ca7b911625b0315ffddfcb8 into 65c79b1932634d53a4c40c528cf842ef(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:30,412 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:30,412 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., storeName=23525a0a9ca7b911625b0315ffddfcb8/C, priority=13, startTime=1731831329947; duration=0sec 2024-11-17T08:15:30,412 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:30,412 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23525a0a9ca7b911625b0315ffddfcb8:C 2024-11-17T08:15:30,451 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/257df833c99c448fbbf7d3e7a298afd3 2024-11-17T08:15:30,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/49d138e8c994496db2daffa36324468d is 50, key is test_row_0/C:col10/1731831329108/Put/seqid=0 2024-11-17T08:15:30,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742109_1285 (size=12151) 2024-11-17T08:15:30,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:30,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:30,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831390583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:30,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831390584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:30,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:30,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:30,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:30,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831390584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:30,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:30,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:30,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831390584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:30,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:30,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:30,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:30,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:30,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:30,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:30,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:30,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:30,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:30,861 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/49d138e8c994496db2daffa36324468d 2024-11-17T08:15:30,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/1f621d86623044388aa9a6e4b1f135d1 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/1f621d86623044388aa9a6e4b1f135d1 2024-11-17T08:15:30,868 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/1f621d86623044388aa9a6e4b1f135d1, entries=150, sequenceid=256, filesize=11.9 K 2024-11-17T08:15:30,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/257df833c99c448fbbf7d3e7a298afd3 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/257df833c99c448fbbf7d3e7a298afd3 2024-11-17T08:15:30,872 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/257df833c99c448fbbf7d3e7a298afd3, entries=150, sequenceid=256, filesize=11.9 K 2024-11-17T08:15:30,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/49d138e8c994496db2daffa36324468d as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/49d138e8c994496db2daffa36324468d 2024-11-17T08:15:30,878 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/49d138e8c994496db2daffa36324468d, entries=150, sequenceid=256, filesize=11.9 K 2024-11-17T08:15:30,878 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 23525a0a9ca7b911625b0315ffddfcb8 in 862ms, sequenceid=256, compaction requested=false 2024-11-17T08:15:30,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:30,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:30,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-17T08:15:30,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-17T08:15:30,881 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-17T08:15:30,881 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7820 sec 2024-11-17T08:15:30,883 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.7870 sec 2024-11-17T08:15:30,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 23525a0a9ca7b911625b0315ffddfcb8 2024-11-17T08:15:30,894 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23525a0a9ca7b911625b0315ffddfcb8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-17T08:15:30,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=A 2024-11-17T08:15:30,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:30,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=B 2024-11-17T08:15:30,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:30,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=C 2024-11-17T08:15:30,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:30,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/3d73084d14b9414eaa5204f13adb0c50 is 50, key is test_row_0/A:col10/1731831330260/Put/seqid=0 2024-11-17T08:15:30,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742110_1286 (size=12301) 2024-11-17T08:15:30,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:30,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831390918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:30,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:30,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:30,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:30,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:30,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:30,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831390918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:30,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831390919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:30,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:30,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:30,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:30,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:30,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831390919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:30,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:30,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:30,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:30,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:30,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:30,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:31,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:31,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831391034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:31,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:31,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:31,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831391034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:31,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:31,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:31,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:31,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:31,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:31,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831391034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:31,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:31,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:31,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:31,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831391034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:31,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:31,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:31,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:31,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:31,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T08:15:31,244 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:15:31,244 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:15:31,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-17T08:15:31,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T08:15:31,246 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:15:31,246 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:15:31,246 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:15:31,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:31,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831391244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:31,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:31,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:31,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:31,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831391244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:31,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:31,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:31,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:31,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:31,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:31,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831391244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:31,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:31,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:31,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:31,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:31,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831391244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:31,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:31,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:31,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:31,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/3d73084d14b9414eaa5204f13adb0c50 2024-11-17T08:15:31,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/eac1a2b64d4a4405b64c2e84aec0a3dd is 50, key is test_row_0/B:col10/1731831330260/Put/seqid=0 2024-11-17T08:15:31,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742111_1287 (size=12301) 2024-11-17T08:15:31,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T08:15:31,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-17T08:15:31,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:31,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. as already flushing 2024-11-17T08:15:31,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:31,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:31,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:31,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:31,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-17T08:15:31,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:31,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. as already flushing 2024-11-17T08:15:31,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:31,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:31,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:31,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:31,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:31,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831391553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:31,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:31,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831391554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:31,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:31,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:31,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:31,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:31,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:31,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:31,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:31,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831391554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:31,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:31,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:31,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:31,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:31,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831391554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:31,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:31,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:31,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T08:15:31,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-17T08:15:31,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:31,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. as already flushing 2024-11-17T08:15:31,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:31,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:31,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:31,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:31,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/eac1a2b64d4a4405b64c2e84aec0a3dd 2024-11-17T08:15:31,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/12f651ca884b4286ae2ac5d0f8fb67c6 is 50, key is test_row_0/C:col10/1731831330260/Put/seqid=0 2024-11-17T08:15:31,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742112_1288 (size=12301) 2024-11-17T08:15:31,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-17T08:15:31,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:31,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. as already flushing 2024-11-17T08:15:31,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:31,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:31,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:31,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:31,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T08:15:32,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-17T08:15:32,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:32,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. as already flushing 2024-11-17T08:15:32,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:32,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:32,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:32,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:32,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:32,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831392065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:32,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:32,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:32,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:32,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:32,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831392065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:32,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:32,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:32,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:32,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:32,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831392065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:32,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:32,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:32,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:32,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:32,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831392065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:32,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:32,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:32,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:32,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/12f651ca884b4286ae2ac5d0f8fb67c6 2024-11-17T08:15:32,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/3d73084d14b9414eaa5204f13adb0c50 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/3d73084d14b9414eaa5204f13adb0c50 2024-11-17T08:15:32,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/3d73084d14b9414eaa5204f13adb0c50, entries=150, sequenceid=271, filesize=12.0 K 2024-11-17T08:15:32,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/eac1a2b64d4a4405b64c2e84aec0a3dd as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/eac1a2b64d4a4405b64c2e84aec0a3dd 2024-11-17T08:15:32,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-17T08:15:32,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:32,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. as already flushing 2024-11-17T08:15:32,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:32,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/eac1a2b64d4a4405b64c2e84aec0a3dd, entries=150, sequenceid=271, filesize=12.0 K 2024-11-17T08:15:32,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:32,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:32,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/12f651ca884b4286ae2ac5d0f8fb67c6 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/12f651ca884b4286ae2ac5d0f8fb67c6 2024-11-17T08:15:32,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:32,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/12f651ca884b4286ae2ac5d0f8fb67c6, entries=150, sequenceid=271, filesize=12.0 K 2024-11-17T08:15:32,177 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 23525a0a9ca7b911625b0315ffddfcb8 in 1283ms, sequenceid=271, compaction requested=true 2024-11-17T08:15:32,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:32,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23525a0a9ca7b911625b0315ffddfcb8:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:15:32,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:32,177 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:15:32,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23525a0a9ca7b911625b0315ffddfcb8:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:15:32,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:32,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23525a0a9ca7b911625b0315ffddfcb8:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:15:32,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:15:32,177 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:15:32,179 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:15:32,179 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:15:32,179 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): 23525a0a9ca7b911625b0315ffddfcb8/B is initiating minor compaction (all files) 2024-11-17T08:15:32,179 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): 23525a0a9ca7b911625b0315ffddfcb8/A is initiating minor compaction (all files) 2024-11-17T08:15:32,179 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23525a0a9ca7b911625b0315ffddfcb8/A in TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:32,179 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23525a0a9ca7b911625b0315ffddfcb8/B in TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:32,179 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/a5c3ec4af12e4e19a7274b68b883f390, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/1f621d86623044388aa9a6e4b1f135d1, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/3d73084d14b9414eaa5204f13adb0c50] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp, totalSize=36.3 K 2024-11-17T08:15:32,179 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/13e8f2586e194b35956507bb09be41e3, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/257df833c99c448fbbf7d3e7a298afd3, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/eac1a2b64d4a4405b64c2e84aec0a3dd] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp, totalSize=36.3 K 2024-11-17T08:15:32,179 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5c3ec4af12e4e19a7274b68b883f390, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731831329084 2024-11-17T08:15:32,179 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 13e8f2586e194b35956507bb09be41e3, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731831329084 2024-11-17T08:15:32,180 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f621d86623044388aa9a6e4b1f135d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731831329108 2024-11-17T08:15:32,180 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 257df833c99c448fbbf7d3e7a298afd3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731831329108 2024-11-17T08:15:32,180 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting eac1a2b64d4a4405b64c2e84aec0a3dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731831330260 2024-11-17T08:15:32,180 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d73084d14b9414eaa5204f13adb0c50, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731831330260 2024-11-17T08:15:32,202 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23525a0a9ca7b911625b0315ffddfcb8#B#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:32,202 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/7f6f0c9890be4ec5847c680d3e273536 is 50, key is test_row_0/B:col10/1731831330260/Put/seqid=0 2024-11-17T08:15:32,208 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23525a0a9ca7b911625b0315ffddfcb8#A#compaction#247 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:32,209 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/d4f322dafeca45d39f569c64d589c043 is 50, key is test_row_0/A:col10/1731831330260/Put/seqid=0 2024-11-17T08:15:32,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742114_1290 (size=13017) 2024-11-17T08:15:32,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742113_1289 (size=13017) 2024-11-17T08:15:32,248 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/d4f322dafeca45d39f569c64d589c043 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/d4f322dafeca45d39f569c64d589c043 2024-11-17T08:15:32,255 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23525a0a9ca7b911625b0315ffddfcb8/A of 23525a0a9ca7b911625b0315ffddfcb8 into d4f322dafeca45d39f569c64d589c043(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:32,255 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:32,255 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., storeName=23525a0a9ca7b911625b0315ffddfcb8/A, priority=13, startTime=1731831332177; duration=0sec 2024-11-17T08:15:32,255 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:15:32,255 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23525a0a9ca7b911625b0315ffddfcb8:A 2024-11-17T08:15:32,256 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:15:32,259 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:15:32,259 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): 23525a0a9ca7b911625b0315ffddfcb8/C is initiating minor compaction (all files) 2024-11-17T08:15:32,259 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23525a0a9ca7b911625b0315ffddfcb8/C in TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:32,259 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/65c79b1932634d53a4c40c528cf842ef, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/49d138e8c994496db2daffa36324468d, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/12f651ca884b4286ae2ac5d0f8fb67c6] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp, totalSize=36.3 K 2024-11-17T08:15:32,260 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65c79b1932634d53a4c40c528cf842ef, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731831329084 2024-11-17T08:15:32,260 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49d138e8c994496db2daffa36324468d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731831329108 2024-11-17T08:15:32,260 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12f651ca884b4286ae2ac5d0f8fb67c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731831330260 2024-11-17T08:15:32,270 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23525a0a9ca7b911625b0315ffddfcb8#C#compaction#248 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:32,271 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/afad2a58aad842c781fade46f17fd2ed is 50, key is test_row_0/C:col10/1731831330260/Put/seqid=0 2024-11-17T08:15:32,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742115_1291 (size=13017) 2024-11-17T08:15:32,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-17T08:15:32,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:32,318 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 23525a0a9ca7b911625b0315ffddfcb8 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-17T08:15:32,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=A 2024-11-17T08:15:32,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:32,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=B 2024-11-17T08:15:32,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:32,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=C 2024-11-17T08:15:32,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:32,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/d0bd8318e22449f3a03a0b65848567cd is 50, key is test_row_0/A:col10/1731831330918/Put/seqid=0 2024-11-17T08:15:32,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742116_1292 (size=12301) 2024-11-17T08:15:32,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T08:15:32,643 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/7f6f0c9890be4ec5847c680d3e273536 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/7f6f0c9890be4ec5847c680d3e273536 2024-11-17T08:15:32,647 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23525a0a9ca7b911625b0315ffddfcb8/B of 23525a0a9ca7b911625b0315ffddfcb8 into 7f6f0c9890be4ec5847c680d3e273536(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:32,647 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:32,647 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., storeName=23525a0a9ca7b911625b0315ffddfcb8/B, priority=13, startTime=1731831332177; duration=0sec 2024-11-17T08:15:32,647 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:32,647 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23525a0a9ca7b911625b0315ffddfcb8:B 2024-11-17T08:15:32,691 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/afad2a58aad842c781fade46f17fd2ed as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/afad2a58aad842c781fade46f17fd2ed 2024-11-17T08:15:32,695 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23525a0a9ca7b911625b0315ffddfcb8/C of 23525a0a9ca7b911625b0315ffddfcb8 into afad2a58aad842c781fade46f17fd2ed(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:32,695 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:32,695 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., storeName=23525a0a9ca7b911625b0315ffddfcb8/C, priority=13, startTime=1731831332177; duration=0sec 2024-11-17T08:15:32,695 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:32,695 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23525a0a9ca7b911625b0315ffddfcb8:C 2024-11-17T08:15:32,735 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/d0bd8318e22449f3a03a0b65848567cd 2024-11-17T08:15:32,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/ce3aa03729594e22b6b7eef91aa5e7c4 is 50, key is test_row_0/B:col10/1731831330918/Put/seqid=0 2024-11-17T08:15:32,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742117_1293 (size=12301) 2024-11-17T08:15:33,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 23525a0a9ca7b911625b0315ffddfcb8 2024-11-17T08:15:33,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. as already flushing 2024-11-17T08:15:33,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:33,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831393093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:33,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:33,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:33,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:33,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:33,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831393094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:33,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:33,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:33,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:33,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:33,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831393096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:33,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:33,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:33,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:33,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:33,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831393098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:33,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:33,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:33,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:33,146 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/ce3aa03729594e22b6b7eef91aa5e7c4 2024-11-17T08:15:33,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/84eed278cec34f4a9860b2443d1b78a0 is 50, key is test_row_0/C:col10/1731831330918/Put/seqid=0 2024-11-17T08:15:33,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742118_1294 (size=12301) 2024-11-17T08:15:33,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:33,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831393204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:33,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:33,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:33,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:33,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831393204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:33,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:33,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:33,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831393204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:33,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:33,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:33,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:33,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:33,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:33,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:33,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:33,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831393213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:33,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:33,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:33,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:33,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T08:15:33,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:33,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831393413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:33,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:33,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831393413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:33,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:33,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:33,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:33,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:33,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:33,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:33,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831393413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:33,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:33,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:33,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:33,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:33,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:33,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831393424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:33,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:33,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:33,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:33,558 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/84eed278cec34f4a9860b2443d1b78a0 2024-11-17T08:15:33,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/d0bd8318e22449f3a03a0b65848567cd as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/d0bd8318e22449f3a03a0b65848567cd 2024-11-17T08:15:33,577 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/d0bd8318e22449f3a03a0b65848567cd, entries=150, sequenceid=291, filesize=12.0 K 2024-11-17T08:15:33,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/ce3aa03729594e22b6b7eef91aa5e7c4 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/ce3aa03729594e22b6b7eef91aa5e7c4 2024-11-17T08:15:33,586 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/ce3aa03729594e22b6b7eef91aa5e7c4, entries=150, sequenceid=291, filesize=12.0 K 2024-11-17T08:15:33,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/84eed278cec34f4a9860b2443d1b78a0 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/84eed278cec34f4a9860b2443d1b78a0 2024-11-17T08:15:33,595 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/84eed278cec34f4a9860b2443d1b78a0, entries=150, sequenceid=291, filesize=12.0 K 2024-11-17T08:15:33,597 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 23525a0a9ca7b911625b0315ffddfcb8 in 1279ms, sequenceid=291, compaction requested=false 2024-11-17T08:15:33,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:33,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:33,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-17T08:15:33,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-17T08:15:33,604 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-17T08:15:33,604 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3540 sec 2024-11-17T08:15:33,607 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 2.3610 sec 2024-11-17T08:15:33,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 23525a0a9ca7b911625b0315ffddfcb8 2024-11-17T08:15:33,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23525a0a9ca7b911625b0315ffddfcb8 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-17T08:15:33,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=A 2024-11-17T08:15:33,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:33,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=B 2024-11-17T08:15:33,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:33,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=C 2024-11-17T08:15:33,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:33,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/3e2a1a973396414286f24c25eb3f75c4 is 50, key is test_row_0/A:col10/1731831333725/Put/seqid=0 2024-11-17T08:15:33,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:33,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831393763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:33,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:33,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831393764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:33,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742119_1295 (size=12301) 2024-11-17T08:15:33,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:33,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831393764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:33,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:33,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:33,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:33,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:33,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831393765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:33,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:33,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:33,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:33,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:33,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:33,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:33,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:33,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:33,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:33,772 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/3e2a1a973396414286f24c25eb3f75c4 2024-11-17T08:15:33,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/a565e8aa7fac4035bbb83a9033663f72 is 50, key is test_row_0/B:col10/1731831333725/Put/seqid=0 2024-11-17T08:15:33,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742120_1296 (size=12301) 2024-11-17T08:15:33,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:33,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831393874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:33,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:33,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:33,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:33,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831393874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:33,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:33,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:33,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:33,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:33,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:33,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831393874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:33,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:33,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:33,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:33,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831393874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:33,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:33,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:33,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:33,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:34,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:34,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831394084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:34,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:34,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:34,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:34,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:34,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831394084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:34,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:34,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:34,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:34,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831394084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:34,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:34,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:34,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:34,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:34,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831394084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:34,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:34,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:34,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:34,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:34,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/a565e8aa7fac4035bbb83a9033663f72 2024-11-17T08:15:34,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/87da5eeb83a3416b8f358d432a5bd1b2 is 50, key is test_row_0/C:col10/1731831333725/Put/seqid=0 2024-11-17T08:15:34,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742121_1297 (size=12301) 2024-11-17T08:15:34,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:34,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831394394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:34,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:34,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:34,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:34,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831394394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:34,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:34,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:34,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:34,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:34,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:34,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831394394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:34,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:34,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:34,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831394394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:34,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:34,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:34,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:34,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:34,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:34,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/87da5eeb83a3416b8f358d432a5bd1b2 2024-11-17T08:15:34,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/3e2a1a973396414286f24c25eb3f75c4 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/3e2a1a973396414286f24c25eb3f75c4 2024-11-17T08:15:34,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/3e2a1a973396414286f24c25eb3f75c4, entries=150, sequenceid=308, filesize=12.0 K 2024-11-17T08:15:34,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/a565e8aa7fac4035bbb83a9033663f72 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/a565e8aa7fac4035bbb83a9033663f72 2024-11-17T08:15:34,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/a565e8aa7fac4035bbb83a9033663f72, entries=150, sequenceid=308, filesize=12.0 K 2024-11-17T08:15:34,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/87da5eeb83a3416b8f358d432a5bd1b2 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/87da5eeb83a3416b8f358d432a5bd1b2 2024-11-17T08:15:34,625 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/87da5eeb83a3416b8f358d432a5bd1b2, entries=150, sequenceid=308, filesize=12.0 K 2024-11-17T08:15:34,626 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 23525a0a9ca7b911625b0315ffddfcb8 in 900ms, sequenceid=308, compaction requested=true 2024-11-17T08:15:34,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:34,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23525a0a9ca7b911625b0315ffddfcb8:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:15:34,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:34,626 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:15:34,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23525a0a9ca7b911625b0315ffddfcb8:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:15:34,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:34,626 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:15:34,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23525a0a9ca7b911625b0315ffddfcb8:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:15:34,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:15:34,628 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:15:34,628 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:15:34,628 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): 23525a0a9ca7b911625b0315ffddfcb8/B is initiating minor compaction (all files) 2024-11-17T08:15:34,628 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): 23525a0a9ca7b911625b0315ffddfcb8/A is initiating minor compaction (all files) 2024-11-17T08:15:34,628 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23525a0a9ca7b911625b0315ffddfcb8/B in TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:34,628 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23525a0a9ca7b911625b0315ffddfcb8/A in TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:34,628 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/7f6f0c9890be4ec5847c680d3e273536, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/ce3aa03729594e22b6b7eef91aa5e7c4, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/a565e8aa7fac4035bbb83a9033663f72] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp, totalSize=36.7 K 2024-11-17T08:15:34,628 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/d4f322dafeca45d39f569c64d589c043, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/d0bd8318e22449f3a03a0b65848567cd, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/3e2a1a973396414286f24c25eb3f75c4] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp, totalSize=36.7 K 2024-11-17T08:15:34,629 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f6f0c9890be4ec5847c680d3e273536, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731831330260 2024-11-17T08:15:34,629 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4f322dafeca45d39f569c64d589c043, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731831330260 2024-11-17T08:15:34,629 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting ce3aa03729594e22b6b7eef91aa5e7c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731831330898 2024-11-17T08:15:34,629 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0bd8318e22449f3a03a0b65848567cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731831330898 2024-11-17T08:15:34,629 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting a565e8aa7fac4035bbb83a9033663f72, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731831333093 2024-11-17T08:15:34,629 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e2a1a973396414286f24c25eb3f75c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731831333093 2024-11-17T08:15:34,637 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23525a0a9ca7b911625b0315ffddfcb8#A#compaction#255 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:34,638 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/1c5ee9d7340944bfbdf9de419eb220ed is 50, key is test_row_0/A:col10/1731831333725/Put/seqid=0 2024-11-17T08:15:34,643 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23525a0a9ca7b911625b0315ffddfcb8#B#compaction#256 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:34,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742122_1298 (size=13119) 2024-11-17T08:15:34,644 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/ea557597fb7e4d96a812eac466f7e641 is 50, key is test_row_0/B:col10/1731831333725/Put/seqid=0 2024-11-17T08:15:34,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742123_1299 (size=13119) 2024-11-17T08:15:34,661 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/ea557597fb7e4d96a812eac466f7e641 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/ea557597fb7e4d96a812eac466f7e641 2024-11-17T08:15:34,671 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23525a0a9ca7b911625b0315ffddfcb8/B of 23525a0a9ca7b911625b0315ffddfcb8 into ea557597fb7e4d96a812eac466f7e641(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:34,671 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:34,671 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., storeName=23525a0a9ca7b911625b0315ffddfcb8/B, priority=13, startTime=1731831334626; duration=0sec 2024-11-17T08:15:34,671 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:15:34,672 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23525a0a9ca7b911625b0315ffddfcb8:B 2024-11-17T08:15:34,672 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:15:34,674 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:15:34,674 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): 23525a0a9ca7b911625b0315ffddfcb8/C is initiating minor compaction (all files) 2024-11-17T08:15:34,674 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23525a0a9ca7b911625b0315ffddfcb8/C in TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:34,674 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/afad2a58aad842c781fade46f17fd2ed, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/84eed278cec34f4a9860b2443d1b78a0, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/87da5eeb83a3416b8f358d432a5bd1b2] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp, totalSize=36.7 K 2024-11-17T08:15:34,675 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting afad2a58aad842c781fade46f17fd2ed, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731831330260 2024-11-17T08:15:34,675 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 84eed278cec34f4a9860b2443d1b78a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731831330898 2024-11-17T08:15:34,676 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 87da5eeb83a3416b8f358d432a5bd1b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731831333093 2024-11-17T08:15:34,684 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23525a0a9ca7b911625b0315ffddfcb8#C#compaction#257 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:34,684 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/4d6dc4e7a6bc4bd28030dc6b1dc93ff0 is 50, key is test_row_0/C:col10/1731831333725/Put/seqid=0 2024-11-17T08:15:34,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742124_1300 (size=13119) 2024-11-17T08:15:34,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 23525a0a9ca7b911625b0315ffddfcb8 2024-11-17T08:15:34,907 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23525a0a9ca7b911625b0315ffddfcb8 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-17T08:15:34,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=A 2024-11-17T08:15:34,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:34,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=B 2024-11-17T08:15:34,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:34,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=C 2024-11-17T08:15:34,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:34,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/de660cc7282345ee9cc223895e7fdef0 is 50, key is test_row_0/A:col10/1731831333763/Put/seqid=0 2024-11-17T08:15:34,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742125_1301 (size=17181) 2024-11-17T08:15:34,926 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/de660cc7282345ee9cc223895e7fdef0 2024-11-17T08:15:34,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:34,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831394924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:34,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:34,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:34,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:34,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831394924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:34,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:34,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:34,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:34,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:34,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:34,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831394926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:34,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:34,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:34,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:34,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:34,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831394928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:34,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:34,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:34,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:34,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/cbce5237275a414ca512f65ee28e291d is 50, key is test_row_0/B:col10/1731831333763/Put/seqid=0 2024-11-17T08:15:34,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742126_1302 (size=12301) 2024-11-17T08:15:35,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:35,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831395033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:35,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:35,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:35,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:35,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831395033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:35,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:35,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:35,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:35,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:35,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:35,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831395034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:35,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:35,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:35,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831395034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:35,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:35,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:35,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:35,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:35,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:35,051 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/1c5ee9d7340944bfbdf9de419eb220ed as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/1c5ee9d7340944bfbdf9de419eb220ed 2024-11-17T08:15:35,055 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23525a0a9ca7b911625b0315ffddfcb8/A of 23525a0a9ca7b911625b0315ffddfcb8 into 1c5ee9d7340944bfbdf9de419eb220ed(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:35,055 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:35,055 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., storeName=23525a0a9ca7b911625b0315ffddfcb8/A, priority=13, startTime=1731831334626; duration=0sec 2024-11-17T08:15:35,056 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:35,056 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23525a0a9ca7b911625b0315ffddfcb8:A 2024-11-17T08:15:35,095 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/4d6dc4e7a6bc4bd28030dc6b1dc93ff0 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/4d6dc4e7a6bc4bd28030dc6b1dc93ff0 2024-11-17T08:15:35,099 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23525a0a9ca7b911625b0315ffddfcb8/C of 23525a0a9ca7b911625b0315ffddfcb8 into 4d6dc4e7a6bc4bd28030dc6b1dc93ff0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:35,099 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:35,099 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., storeName=23525a0a9ca7b911625b0315ffddfcb8/C, priority=13, startTime=1731831334626; duration=0sec 2024-11-17T08:15:35,099 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:35,099 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23525a0a9ca7b911625b0315ffddfcb8:C 2024-11-17T08:15:35,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:35,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831395243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:35,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:35,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831395243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:35,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:35,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:35,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:35,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831395243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:35,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:35,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:35,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:35,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:35,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:35,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:35,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831395243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:35,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:35,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:35,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:35,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:35,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:35,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/cbce5237275a414ca512f65ee28e291d 2024-11-17T08:15:35,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/4de51a14e6b041d5a85a0c8bdf743698 is 50, key is test_row_0/C:col10/1731831333763/Put/seqid=0 2024-11-17T08:15:35,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742127_1303 (size=12301) 2024-11-17T08:15:35,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T08:15:35,414 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:15:35,415 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:15:35,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-17T08:15:35,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-17T08:15:35,417 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:15:35,417 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:15:35,418 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:15:35,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-17T08:15:35,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:35,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831395553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:35,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:35,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:35,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:35,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:35,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831395553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:35,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:35,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:35,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:35,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:35,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831395553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:35,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:35,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:35,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:35,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831395553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:35,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:35,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:35,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:35,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:35,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-17T08:15:35,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:35,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. as already flushing 2024-11-17T08:15:35,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:35,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:35,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:35,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:35,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-17T08:15:35,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:35,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. as already flushing 2024-11-17T08:15:35,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:35,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:35,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:35,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:35,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-17T08:15:35,765 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/4de51a14e6b041d5a85a0c8bdf743698 2024-11-17T08:15:35,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/de660cc7282345ee9cc223895e7fdef0 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/de660cc7282345ee9cc223895e7fdef0 2024-11-17T08:15:35,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/de660cc7282345ee9cc223895e7fdef0, entries=250, sequenceid=328, filesize=16.8 K 2024-11-17T08:15:35,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/cbce5237275a414ca512f65ee28e291d as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/cbce5237275a414ca512f65ee28e291d 2024-11-17T08:15:35,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/cbce5237275a414ca512f65ee28e291d, entries=150, sequenceid=328, filesize=12.0 K 2024-11-17T08:15:35,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/4de51a14e6b041d5a85a0c8bdf743698 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/4de51a14e6b041d5a85a0c8bdf743698 2024-11-17T08:15:35,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/4de51a14e6b041d5a85a0c8bdf743698, entries=150, sequenceid=328, filesize=12.0 K 2024-11-17T08:15:35,783 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 23525a0a9ca7b911625b0315ffddfcb8 in 876ms, sequenceid=328, compaction requested=false 2024-11-17T08:15:35,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:35,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-17T08:15:35,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:35,876 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 23525a0a9ca7b911625b0315ffddfcb8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-17T08:15:35,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=A 2024-11-17T08:15:35,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:35,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=B 2024-11-17T08:15:35,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:35,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=C 2024-11-17T08:15:35,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:35,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/a8b779504e7e47608bfef186329ffed8 is 50, key is test_row_0/A:col10/1731831334924/Put/seqid=0 2024-11-17T08:15:35,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742128_1304 (size=12301) 2024-11-17T08:15:35,908 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/a8b779504e7e47608bfef186329ffed8 2024-11-17T08:15:35,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/235373f8d06d4995bd05d5b5cbe7b53a is 50, key is test_row_0/B:col10/1731831334924/Put/seqid=0 2024-11-17T08:15:36,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742129_1305 (size=12301) 2024-11-17T08:15:36,009 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/235373f8d06d4995bd05d5b5cbe7b53a 2024-11-17T08:15:36,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/c6a13d53394d4120ac4c3952b2a931a7 is 50, key is test_row_0/C:col10/1731831334924/Put/seqid=0 2024-11-17T08:15:36,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742130_1306 (size=12301) 2024-11-17T08:15:36,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-17T08:15:36,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 23525a0a9ca7b911625b0315ffddfcb8 2024-11-17T08:15:36,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. as already flushing 2024-11-17T08:15:36,111 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T08:15:36,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:36,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831396112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:36,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:36,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:36,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:36,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831396113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:36,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:36,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:36,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:36,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:36,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:36,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831396114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:36,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:36,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831396114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:36,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:36,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:36,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:36,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:36,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:36,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:36,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:36,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831396224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:36,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:36,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:36,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:36,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:36,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831396224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:36,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:36,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:36,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:36,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:36,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831396224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:36,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:36,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:36,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:36,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:36,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831396225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:36,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:36,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:36,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:36,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:36,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831396434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:36,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:36,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831396435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:36,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:36,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:36,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:36,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:36,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:36,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:36,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:36,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831396435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:36,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:36,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:36,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:36,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:36,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831396435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:36,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:36,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:36,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:36,442 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/c6a13d53394d4120ac4c3952b2a931a7 2024-11-17T08:15:36,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/a8b779504e7e47608bfef186329ffed8 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/a8b779504e7e47608bfef186329ffed8 2024-11-17T08:15:36,456 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/a8b779504e7e47608bfef186329ffed8, entries=150, sequenceid=343, filesize=12.0 K 2024-11-17T08:15:36,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/235373f8d06d4995bd05d5b5cbe7b53a as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/235373f8d06d4995bd05d5b5cbe7b53a 2024-11-17T08:15:36,463 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/235373f8d06d4995bd05d5b5cbe7b53a, entries=150, sequenceid=343, filesize=12.0 K 2024-11-17T08:15:36,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/c6a13d53394d4120ac4c3952b2a931a7 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/c6a13d53394d4120ac4c3952b2a931a7 2024-11-17T08:15:36,471 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/c6a13d53394d4120ac4c3952b2a931a7, entries=150, sequenceid=343, filesize=12.0 K 2024-11-17T08:15:36,474 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 23525a0a9ca7b911625b0315ffddfcb8 in 598ms, sequenceid=343, compaction requested=true 2024-11-17T08:15:36,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:36,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:36,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-17T08:15:36,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-17T08:15:36,478 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-17T08:15:36,478 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0590 sec 2024-11-17T08:15:36,482 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.0650 sec 2024-11-17T08:15:36,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-17T08:15:36,554 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:15:36,555 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:15:36,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-17T08:15:36,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-17T08:15:36,558 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:15:36,559 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:15:36,559 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:15:36,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-17T08:15:36,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-17T08:15:36,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:36,714 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing 23525a0a9ca7b911625b0315ffddfcb8 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-17T08:15:36,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=A 2024-11-17T08:15:36,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:36,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=B 2024-11-17T08:15:36,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:36,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=C 2024-11-17T08:15:36,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:36,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/59914c4f2b1e4266972bb2e3da8a1892 is 50, key is test_row_0/A:col10/1731831336111/Put/seqid=0 2024-11-17T08:15:36,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 23525a0a9ca7b911625b0315ffddfcb8 2024-11-17T08:15:36,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. as already flushing 2024-11-17T08:15:36,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742131_1307 (size=12301) 2024-11-17T08:15:36,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:36,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831396763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:36,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:36,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:36,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:36,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:36,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831396768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:36,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:36,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:36,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:36,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:36,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831396765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:36,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:36,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:36,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:36,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:36,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831396768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:36,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:36,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:36,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:36,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-17T08:15:36,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:36,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831396877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:36,881 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:36,881 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:36,881 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:36,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:36,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831396884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:36,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:36,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:36,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:36,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:36,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831396884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:36,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:36,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:36,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831396884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:36,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:36,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:36,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:36,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:36,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:37,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:37,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831397083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:37,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:37,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:37,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:37,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:37,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831397094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:37,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:37,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:37,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:37,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:37,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831397094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:37,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:37,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:37,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:37,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:37,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831397094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:37,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:37,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:37,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:37,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:37,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53876 deadline: 1731831397103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:37,105 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to e3d0adb87c01,40631,1731831247805 for 'test_row_2' in 23525a0a9ca7b911625b0315ffddfcb8 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T08:15:37,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:37,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:37,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:37,166 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/59914c4f2b1e4266972bb2e3da8a1892 2024-11-17T08:15:37,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/8589b505e79541f48cd3f225f84b6dc1 is 50, key is test_row_0/B:col10/1731831336111/Put/seqid=0 2024-11-17T08:15:37,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-17T08:15:37,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742132_1308 (size=12301) 2024-11-17T08:15:37,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:37,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831397394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:37,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:37,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:37,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:37,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:37,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831397404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:37,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:37,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831397405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:37,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:37,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:37,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:37,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:37,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:37,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:37,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:37,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831397414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:37,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:37,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:37,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:37,591 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/8589b505e79541f48cd3f225f84b6dc1 2024-11-17T08:15:37,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/b741a0c625c448608e37d1afa8a0ca5f is 50, key is test_row_0/C:col10/1731831336111/Put/seqid=0 2024-11-17T08:15:37,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742133_1309 (size=12301) 2024-11-17T08:15:37,622 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/b741a0c625c448608e37d1afa8a0ca5f 2024-11-17T08:15:37,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/59914c4f2b1e4266972bb2e3da8a1892 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/59914c4f2b1e4266972bb2e3da8a1892 2024-11-17T08:15:37,634 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/59914c4f2b1e4266972bb2e3da8a1892, entries=150, sequenceid=362, filesize=12.0 K 2024-11-17T08:15:37,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/8589b505e79541f48cd3f225f84b6dc1 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/8589b505e79541f48cd3f225f84b6dc1 2024-11-17T08:15:37,643 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/8589b505e79541f48cd3f225f84b6dc1, entries=150, sequenceid=362, filesize=12.0 K 2024-11-17T08:15:37,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/b741a0c625c448608e37d1afa8a0ca5f as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/b741a0c625c448608e37d1afa8a0ca5f 2024-11-17T08:15:37,650 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/b741a0c625c448608e37d1afa8a0ca5f, entries=150, sequenceid=362, filesize=12.0 K 2024-11-17T08:15:37,653 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 23525a0a9ca7b911625b0315ffddfcb8 in 939ms, sequenceid=362, compaction requested=true 2024-11-17T08:15:37,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:37,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:37,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-17T08:15:37,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-17T08:15:37,656 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-17T08:15:37,656 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0960 sec 2024-11-17T08:15:37,658 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 1.1020 sec 2024-11-17T08:15:37,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-17T08:15:37,693 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:15:37,694 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:15:37,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-17T08:15:37,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-17T08:15:37,695 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:15:37,696 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:15:37,696 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:15:37,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-17T08:15:37,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-17T08:15:37,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:37,849 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing 23525a0a9ca7b911625b0315ffddfcb8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-17T08:15:37,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=A 2024-11-17T08:15:37,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:37,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=B 2024-11-17T08:15:37,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:37,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=C 2024-11-17T08:15:37,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:37,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/a64edaeabb3c4cdf9717af4c1b456c88 is 50, key is test_row_0/A:col10/1731831336766/Put/seqid=0 2024-11-17T08:15:37,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742134_1310 (size=12301) 2024-11-17T08:15:37,894 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/a64edaeabb3c4cdf9717af4c1b456c88 2024-11-17T08:15:37,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/085573d53cdc4dc0bdd9880442e2300e is 50, key is test_row_0/B:col10/1731831336766/Put/seqid=0 2024-11-17T08:15:37,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742135_1311 (size=12301) 2024-11-17T08:15:37,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 23525a0a9ca7b911625b0315ffddfcb8 2024-11-17T08:15:37,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. as already flushing 2024-11-17T08:15:37,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:37,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831397948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:37,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:37,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:37,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:37,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:37,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831397951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:37,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:37,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:37,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:37,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831397952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:37,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:37,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:37,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:37,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831397952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:37,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:37,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:37,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:37,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:37,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:38,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-17T08:15:38,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:38,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831398064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:38,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:38,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:38,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:38,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:38,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831398064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:38,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:38,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:38,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:38,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831398064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:38,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:38,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:38,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:38,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:38,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:38,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831398064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:38,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:38,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:38,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:38,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:38,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831398273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:38,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:38,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:38,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:38,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831398273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:38,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:38,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:38,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:38,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:38,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:38,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831398273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:38,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:38,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:38,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:38,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:38,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831398273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:38,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:38,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:38,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:38,317 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/085573d53cdc4dc0bdd9880442e2300e 2024-11-17T08:15:38,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/44ea2d3aa9cd43fd8e368deb5b992a2c is 50, key is test_row_0/C:col10/1731831336766/Put/seqid=0 2024-11-17T08:15:38,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-17T08:15:38,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742136_1312 (size=12301) 2024-11-17T08:15:38,334 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/44ea2d3aa9cd43fd8e368deb5b992a2c 2024-11-17T08:15:38,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/a64edaeabb3c4cdf9717af4c1b456c88 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/a64edaeabb3c4cdf9717af4c1b456c88 2024-11-17T08:15:38,343 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/a64edaeabb3c4cdf9717af4c1b456c88, entries=150, sequenceid=375, filesize=12.0 K 2024-11-17T08:15:38,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/085573d53cdc4dc0bdd9880442e2300e as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/085573d53cdc4dc0bdd9880442e2300e 2024-11-17T08:15:38,349 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/085573d53cdc4dc0bdd9880442e2300e, entries=150, sequenceid=375, filesize=12.0 K 2024-11-17T08:15:38,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/44ea2d3aa9cd43fd8e368deb5b992a2c as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/44ea2d3aa9cd43fd8e368deb5b992a2c 2024-11-17T08:15:38,354 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/44ea2d3aa9cd43fd8e368deb5b992a2c, entries=150, sequenceid=375, filesize=12.0 K 2024-11-17T08:15:38,355 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 23525a0a9ca7b911625b0315ffddfcb8 in 506ms, sequenceid=375, compaction requested=true 2024-11-17T08:15:38,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:38,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:38,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-17T08:15:38,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-17T08:15:38,358 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-17T08:15:38,358 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 661 msec 2024-11-17T08:15:38,360 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 665 msec 2024-11-17T08:15:38,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 23525a0a9ca7b911625b0315ffddfcb8 2024-11-17T08:15:38,585 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23525a0a9ca7b911625b0315ffddfcb8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-17T08:15:38,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=A 2024-11-17T08:15:38,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:38,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=B 2024-11-17T08:15:38,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:38,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=C 2024-11-17T08:15:38,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:38,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/bd868fb6188f409fa30f13f467352b67 is 50, key is test_row_0/A:col10/1731831337951/Put/seqid=0 2024-11-17T08:15:38,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742137_1313 (size=14741) 2024-11-17T08:15:38,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/bd868fb6188f409fa30f13f467352b67 2024-11-17T08:15:38,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/c83e41b1e56144e09fafc0c0f9b7c604 is 50, key is test_row_0/B:col10/1731831337951/Put/seqid=0 2024-11-17T08:15:38,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742138_1314 (size=12301) 2024-11-17T08:15:38,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:38,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831398597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:38,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:38,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:38,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:38,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:38,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831398603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:38,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:38,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:38,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831398603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:38,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:38,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:38,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:38,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:38,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:38,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:38,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831398604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:38,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:38,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:38,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:38,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:38,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831398713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:38,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:38,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:38,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:38,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:38,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831398714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:38,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:38,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831398714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:38,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:38,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:38,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:38,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831398714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:38,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:38,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:38,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:38,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:38,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:38,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:38,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:38,765 INFO [Thread-1073 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1073. 2024-11-17T08:15:38,765 DEBUG [Thread-1073 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:15:38,766 DEBUG [Thread-1073 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:38,766 DEBUG [Thread-1073 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:38,766 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:15:38,768 INFO [Thread-1079 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1079. 2024-11-17T08:15:38,768 DEBUG [Thread-1079 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:15:38,768 DEBUG [Thread-1079 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:38,768 DEBUG [Thread-1079 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:38,769 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:15:38,770 INFO [Thread-1071 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1071. 2024-11-17T08:15:38,770 DEBUG [Thread-1071 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:15:38,770 DEBUG [Thread-1071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:38,770 DEBUG [Thread-1071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:38,770 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:15:38,771 INFO [Thread-1077 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1077. 2024-11-17T08:15:38,771 DEBUG [Thread-1077 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:15:38,771 DEBUG [Thread-1077 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:38,771 DEBUG [Thread-1077 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:38,771 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:15:38,773 INFO [Thread-1075 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1075. 2024-11-17T08:15:38,773 DEBUG [Thread-1075 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:15:38,773 DEBUG [Thread-1075 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:38,773 DEBUG [Thread-1075 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:38,773 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:15:38,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-17T08:15:38,834 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:15:38,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:38,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:38,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:38,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831398924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:38,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831398924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:38,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831398924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:38,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:38,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:38,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:38,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:38,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:38,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:38,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:38,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:38,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:38,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:38,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831398933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:38,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:38,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:38,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:39,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/c83e41b1e56144e09fafc0c0f9b7c604 2024-11-17T08:15:39,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/e2d4bd392c204c3487a38a3c0da6ab52 is 50, key is test_row_0/C:col10/1731831337951/Put/seqid=0 2024-11-17T08:15:39,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742139_1315 (size=12301) 2024-11-17T08:15:39,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:39,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:39,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53874 deadline: 1731831399234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:39,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:39,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53860 deadline: 1731831399234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:39,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53810 deadline: 1731831399234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:39,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:39,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:39,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:39,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:39,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:39,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:39,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:39,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:39,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:39,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:39,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53830 deadline: 1731831399243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:39,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:39,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:39,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:15:39,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/e2d4bd392c204c3487a38a3c0da6ab52 2024-11-17T08:15:39,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/bd868fb6188f409fa30f13f467352b67 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/bd868fb6188f409fa30f13f467352b67 2024-11-17T08:15:39,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/bd868fb6188f409fa30f13f467352b67, entries=200, sequenceid=395, filesize=14.4 K 2024-11-17T08:15:39,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/c83e41b1e56144e09fafc0c0f9b7c604 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/c83e41b1e56144e09fafc0c0f9b7c604 2024-11-17T08:15:39,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/c83e41b1e56144e09fafc0c0f9b7c604, entries=150, sequenceid=395, filesize=12.0 K 2024-11-17T08:15:39,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/e2d4bd392c204c3487a38a3c0da6ab52 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/e2d4bd392c204c3487a38a3c0da6ab52 2024-11-17T08:15:39,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/e2d4bd392c204c3487a38a3c0da6ab52, entries=150, sequenceid=395, filesize=12.0 K 2024-11-17T08:15:39,445 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 23525a0a9ca7b911625b0315ffddfcb8 in 860ms, sequenceid=395, compaction requested=true 2024-11-17T08:15:39,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:39,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23525a0a9ca7b911625b0315ffddfcb8:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:15:39,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:39,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23525a0a9ca7b911625b0315ffddfcb8:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:15:39,445 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-17T08:15:39,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:15:39,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23525a0a9ca7b911625b0315ffddfcb8:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:15:39,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:15:39,445 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-17T08:15:39,446 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 81944 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-17T08:15:39,446 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 74624 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-17T08:15:39,446 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): 23525a0a9ca7b911625b0315ffddfcb8/B is initiating minor compaction (all files) 2024-11-17T08:15:39,446 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): 23525a0a9ca7b911625b0315ffddfcb8/A is initiating minor compaction (all files) 2024-11-17T08:15:39,446 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23525a0a9ca7b911625b0315ffddfcb8/A in TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:39,446 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23525a0a9ca7b911625b0315ffddfcb8/B in TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:39,446 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/1c5ee9d7340944bfbdf9de419eb220ed, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/de660cc7282345ee9cc223895e7fdef0, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/a8b779504e7e47608bfef186329ffed8, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/59914c4f2b1e4266972bb2e3da8a1892, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/a64edaeabb3c4cdf9717af4c1b456c88, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/bd868fb6188f409fa30f13f467352b67] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp, totalSize=80.0 K 2024-11-17T08:15:39,446 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/ea557597fb7e4d96a812eac466f7e641, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/cbce5237275a414ca512f65ee28e291d, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/235373f8d06d4995bd05d5b5cbe7b53a, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/8589b505e79541f48cd3f225f84b6dc1, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/085573d53cdc4dc0bdd9880442e2300e, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/c83e41b1e56144e09fafc0c0f9b7c604] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp, totalSize=72.9 K 2024-11-17T08:15:39,447 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c5ee9d7340944bfbdf9de419eb220ed, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731831333093 2024-11-17T08:15:39,447 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting ea557597fb7e4d96a812eac466f7e641, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731831333093 2024-11-17T08:15:39,447 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting de660cc7282345ee9cc223895e7fdef0, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731831333745 2024-11-17T08:15:39,447 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting cbce5237275a414ca512f65ee28e291d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731831333745 2024-11-17T08:15:39,447 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 235373f8d06d4995bd05d5b5cbe7b53a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731831334913 2024-11-17T08:15:39,447 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8b779504e7e47608bfef186329ffed8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731831334913 2024-11-17T08:15:39,448 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 8589b505e79541f48cd3f225f84b6dc1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731831336111 2024-11-17T08:15:39,448 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59914c4f2b1e4266972bb2e3da8a1892, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731831336111 2024-11-17T08:15:39,448 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 085573d53cdc4dc0bdd9880442e2300e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731831336763 2024-11-17T08:15:39,448 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting a64edaeabb3c4cdf9717af4c1b456c88, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731831336763 2024-11-17T08:15:39,448 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd868fb6188f409fa30f13f467352b67, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731831337949 2024-11-17T08:15:39,448 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting c83e41b1e56144e09fafc0c0f9b7c604, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731831337949 2024-11-17T08:15:39,459 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23525a0a9ca7b911625b0315ffddfcb8#B#compaction#273 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:39,459 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23525a0a9ca7b911625b0315ffddfcb8#A#compaction#274 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:39,460 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/07a2ff92f82b47ad8c4f65ccb810b1bb is 50, key is test_row_0/B:col10/1731831337951/Put/seqid=0 2024-11-17T08:15:39,460 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/86dbeb0f0f23457bbd97e015a72c8457 is 50, key is test_row_0/A:col10/1731831337951/Put/seqid=0 2024-11-17T08:15:39,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742141_1317 (size=13323) 2024-11-17T08:15:39,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742140_1316 (size=13323) 2024-11-17T08:15:39,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 23525a0a9ca7b911625b0315ffddfcb8 2024-11-17T08:15:39,748 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23525a0a9ca7b911625b0315ffddfcb8 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-17T08:15:39,749 INFO [Thread-1068 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1068. 2024-11-17T08:15:39,749 DEBUG [Thread-1068 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:15:39,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=A 2024-11-17T08:15:39,749 DEBUG [Thread-1068 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:39,749 INFO [Thread-1060 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1060. 2024-11-17T08:15:39,749 INFO [Thread-1064 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1064. 2024-11-17T08:15:39,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:39,749 DEBUG [Thread-1060 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:15:39,749 DEBUG [Thread-1064 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:15:39,749 DEBUG [Thread-1068 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:39,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=B 2024-11-17T08:15:39,749 DEBUG [Thread-1060 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:39,749 DEBUG [Thread-1064 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:39,749 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:15:39,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:39,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=C 2024-11-17T08:15:39,749 DEBUG [Thread-1064 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:39,749 DEBUG [Thread-1060 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:39,750 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:15:39,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:39,750 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:15:39,755 INFO [Thread-1062 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1062. 2024-11-17T08:15:39,755 DEBUG [Thread-1062 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:15:39,755 DEBUG [Thread-1062 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:39,755 DEBUG [Thread-1062 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:39,755 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:15:39,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/4d0091bef6e341698aa09077aa473fe4 is 50, key is test_row_0/A:col10/1731831339745/Put/seqid=0 2024-11-17T08:15:39,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742142_1318 (size=12301) 2024-11-17T08:15:39,873 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/07a2ff92f82b47ad8c4f65ccb810b1bb as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/07a2ff92f82b47ad8c4f65ccb810b1bb 2024-11-17T08:15:39,873 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/86dbeb0f0f23457bbd97e015a72c8457 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/86dbeb0f0f23457bbd97e015a72c8457 2024-11-17T08:15:39,879 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 23525a0a9ca7b911625b0315ffddfcb8/A of 23525a0a9ca7b911625b0315ffddfcb8 into 86dbeb0f0f23457bbd97e015a72c8457(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:39,879 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 23525a0a9ca7b911625b0315ffddfcb8/B of 23525a0a9ca7b911625b0315ffddfcb8 into 07a2ff92f82b47ad8c4f65ccb810b1bb(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:39,879 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:39,879 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:39,879 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., storeName=23525a0a9ca7b911625b0315ffddfcb8/B, priority=10, startTime=1731831339445; duration=0sec 2024-11-17T08:15:39,879 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., storeName=23525a0a9ca7b911625b0315ffddfcb8/A, priority=10, startTime=1731831339445; duration=0sec 2024-11-17T08:15:39,879 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:15:39,879 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23525a0a9ca7b911625b0315ffddfcb8:B 2024-11-17T08:15:39,879 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:15:39,879 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23525a0a9ca7b911625b0315ffddfcb8:A 2024-11-17T08:15:39,879 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-17T08:15:39,881 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 74624 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-17T08:15:39,881 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): 23525a0a9ca7b911625b0315ffddfcb8/C is initiating minor compaction (all files) 2024-11-17T08:15:39,881 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23525a0a9ca7b911625b0315ffddfcb8/C in TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:39,881 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/4d6dc4e7a6bc4bd28030dc6b1dc93ff0, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/4de51a14e6b041d5a85a0c8bdf743698, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/c6a13d53394d4120ac4c3952b2a931a7, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/b741a0c625c448608e37d1afa8a0ca5f, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/44ea2d3aa9cd43fd8e368deb5b992a2c, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/e2d4bd392c204c3487a38a3c0da6ab52] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp, totalSize=72.9 K 2024-11-17T08:15:39,881 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d6dc4e7a6bc4bd28030dc6b1dc93ff0, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731831333093 2024-11-17T08:15:39,882 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 4de51a14e6b041d5a85a0c8bdf743698, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731831333745 2024-11-17T08:15:39,882 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting c6a13d53394d4120ac4c3952b2a931a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731831334913 2024-11-17T08:15:39,882 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting b741a0c625c448608e37d1afa8a0ca5f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731831336111 2024-11-17T08:15:39,883 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 44ea2d3aa9cd43fd8e368deb5b992a2c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731831336763 2024-11-17T08:15:39,883 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting e2d4bd392c204c3487a38a3c0da6ab52, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731831337949 2024-11-17T08:15:39,893 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23525a0a9ca7b911625b0315ffddfcb8#C#compaction#276 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:39,894 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/603f3e7afd3f48fa88522a94fd88ba84 is 50, key is test_row_0/C:col10/1731831337951/Put/seqid=0 2024-11-17T08:15:39,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742143_1319 (size=13323) 2024-11-17T08:15:40,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/4d0091bef6e341698aa09077aa473fe4 2024-11-17T08:15:40,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/cb6688a264a84b05b2df3ed81325774a is 50, key is test_row_0/B:col10/1731831339745/Put/seqid=0 2024-11-17T08:15:40,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742144_1320 (size=12301) 2024-11-17T08:15:40,302 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/603f3e7afd3f48fa88522a94fd88ba84 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/603f3e7afd3f48fa88522a94fd88ba84 2024-11-17T08:15:40,308 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 23525a0a9ca7b911625b0315ffddfcb8/C of 23525a0a9ca7b911625b0315ffddfcb8 into 603f3e7afd3f48fa88522a94fd88ba84(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:40,308 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:40,308 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., storeName=23525a0a9ca7b911625b0315ffddfcb8/C, priority=10, startTime=1731831339445; duration=0sec 2024-11-17T08:15:40,308 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:40,308 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23525a0a9ca7b911625b0315ffddfcb8:C 2024-11-17T08:15:40,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/cb6688a264a84b05b2df3ed81325774a 2024-11-17T08:15:40,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/6ef92cb51925426c83fe2cfd6df86b39 is 50, key is test_row_0/C:col10/1731831339745/Put/seqid=0 2024-11-17T08:15:40,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742145_1321 (size=12301) 2024-11-17T08:15:40,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/6ef92cb51925426c83fe2cfd6df86b39 2024-11-17T08:15:40,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/4d0091bef6e341698aa09077aa473fe4 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/4d0091bef6e341698aa09077aa473fe4 2024-11-17T08:15:40,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/4d0091bef6e341698aa09077aa473fe4, entries=150, sequenceid=411, filesize=12.0 K 2024-11-17T08:15:40,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/cb6688a264a84b05b2df3ed81325774a as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/cb6688a264a84b05b2df3ed81325774a 2024-11-17T08:15:40,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/cb6688a264a84b05b2df3ed81325774a, entries=150, sequenceid=411, filesize=12.0 K 2024-11-17T08:15:40,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/6ef92cb51925426c83fe2cfd6df86b39 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/6ef92cb51925426c83fe2cfd6df86b39 2024-11-17T08:15:40,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/6ef92cb51925426c83fe2cfd6df86b39, entries=150, sequenceid=411, filesize=12.0 K 2024-11-17T08:15:40,615 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=6.71 KB/6870 for 23525a0a9ca7b911625b0315ffddfcb8 in 867ms, sequenceid=411, compaction requested=false 2024-11-17T08:15:40,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23525a0a9ca7b911625b0315ffddfcb8: 2024-11-17T08:15:47,189 INFO [Thread-1066 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1066. 2024-11-17T08:15:47,189 DEBUG [Thread-1066 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:15:47,189 DEBUG [Thread-1066 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:47,189 DEBUG [Thread-1066 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:47,190 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:15:47,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T08:15:47,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-17T08:15:47,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-17T08:15:47,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-17T08:15:47,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 6 2024-11-17T08:15:47,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-17T08:15:47,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T08:15:47,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3165 2024-11-17T08:15:47,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3193 2024-11-17T08:15:47,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3192 2024-11-17T08:15:47,190 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3155 2024-11-17T08:15:47,191 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3208 2024-11-17T08:15:47,191 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T08:15:47,191 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T08:15:47,191 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T08:15:47,191 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:47,191 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:47,191 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:15:47,193 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-17T08:15:47,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T08:15:47,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-17T08:15:47,199 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731831347199"}]},"ts":"1731831347199"} 2024-11-17T08:15:47,201 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T08:15:47,201 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T08:15:47,201 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T08:15:47,203 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=23525a0a9ca7b911625b0315ffddfcb8, UNASSIGN}] 2024-11-17T08:15:47,203 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=23525a0a9ca7b911625b0315ffddfcb8, UNASSIGN 2024-11-17T08:15:47,204 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=23525a0a9ca7b911625b0315ffddfcb8, regionState=CLOSING, regionLocation=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:47,205 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=23525a0a9ca7b911625b0315ffddfcb8, UNASSIGN because future has completed 2024-11-17T08:15:47,205 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T08:15:47,205 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805}] 2024-11-17T08:15:47,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-17T08:15:47,357 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close 23525a0a9ca7b911625b0315ffddfcb8 2024-11-17T08:15:47,357 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T08:15:47,358 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing 23525a0a9ca7b911625b0315ffddfcb8, disabling compactions & flushes 2024-11-17T08:15:47,358 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:47,358 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:47,358 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. after waiting 0 ms 2024-11-17T08:15:47,358 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:47,358 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(2902): Flushing 23525a0a9ca7b911625b0315ffddfcb8 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-17T08:15:47,358 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=A 2024-11-17T08:15:47,358 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:47,358 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=B 2024-11-17T08:15:47,358 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:47,358 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 23525a0a9ca7b911625b0315ffddfcb8, store=C 2024-11-17T08:15:47,358 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:47,362 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/296088fc160a4421913eed3f08315b1e is 50, key is test_row_2/A:col10/1731831347185/Put/seqid=0 2024-11-17T08:15:47,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742146_1322 (size=7415) 2024-11-17T08:15:47,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-17T08:15:47,765 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/296088fc160a4421913eed3f08315b1e 2024-11-17T08:15:47,772 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/c8ce5eef9caa4ae099382596bdb8d2af is 50, key is test_row_2/B:col10/1731831347185/Put/seqid=0 2024-11-17T08:15:47,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742147_1323 (size=7415) 2024-11-17T08:15:47,781 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/c8ce5eef9caa4ae099382596bdb8d2af 2024-11-17T08:15:47,788 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/5d5f371216104e82996ce83a17a74aba is 50, key is test_row_2/C:col10/1731831347185/Put/seqid=0 2024-11-17T08:15:47,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742148_1324 (size=7415) 2024-11-17T08:15:47,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-17T08:15:48,192 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/5d5f371216104e82996ce83a17a74aba 2024-11-17T08:15:48,196 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/A/296088fc160a4421913eed3f08315b1e as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/296088fc160a4421913eed3f08315b1e 2024-11-17T08:15:48,199 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/296088fc160a4421913eed3f08315b1e, entries=50, sequenceid=419, filesize=7.2 K 2024-11-17T08:15:48,200 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/B/c8ce5eef9caa4ae099382596bdb8d2af as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/c8ce5eef9caa4ae099382596bdb8d2af 2024-11-17T08:15:48,203 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/c8ce5eef9caa4ae099382596bdb8d2af, entries=50, sequenceid=419, filesize=7.2 K 2024-11-17T08:15:48,204 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/.tmp/C/5d5f371216104e82996ce83a17a74aba as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/5d5f371216104e82996ce83a17a74aba 2024-11-17T08:15:48,206 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/5d5f371216104e82996ce83a17a74aba, entries=50, sequenceid=419, filesize=7.2 K 2024-11-17T08:15:48,207 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 23525a0a9ca7b911625b0315ffddfcb8 in 849ms, sequenceid=419, compaction requested=true 2024-11-17T08:15:48,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/75b633cb4ce64f1faea17a12b70eccf7, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/6890b7afd8124947a2a8feba3307f071, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/818e0eccbebd4b978e3584727e5aed1e, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/2e781ec865144a6b893f9d42e0cd87b7, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/0c7330ebde3c432099230a3932394504, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/84001bdf570d4fcd8f71211646f87260, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/905c356fe62f44c8a4a9d80c967db163, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/0e985e982d66493cb376e07c3cf301b8, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/8486c769e35b4506ae6befabbb8827c2, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/1e85ad80bf764da0ba214a4e15adf5f3, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/fbaeee0be0ba44ed92d1ee14a6c93394, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/0e99033d738541baaa4249d1865e41b8, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/a4d0c85b8e2244d8b0f520895736d206, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/579ac7af3e4f4d10834963770c34d21d, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/8690451de16c4d1c95790bd167ffda20, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/aec757bb230d4859a08ba36ee671eddc, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/dff2d328b3c647d1a4befe994e205807, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/4569d78750764c1fa49b2e61a97399ce, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/a5c3ec4af12e4e19a7274b68b883f390, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/1f621d86623044388aa9a6e4b1f135d1, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/d4f322dafeca45d39f569c64d589c043, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/3d73084d14b9414eaa5204f13adb0c50, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/d0bd8318e22449f3a03a0b65848567cd, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/1c5ee9d7340944bfbdf9de419eb220ed, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/3e2a1a973396414286f24c25eb3f75c4, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/de660cc7282345ee9cc223895e7fdef0, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/a8b779504e7e47608bfef186329ffed8, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/59914c4f2b1e4266972bb2e3da8a1892, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/a64edaeabb3c4cdf9717af4c1b456c88, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/bd868fb6188f409fa30f13f467352b67] to archive 2024-11-17T08:15:48,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T08:15:48,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/75b633cb4ce64f1faea17a12b70eccf7 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/75b633cb4ce64f1faea17a12b70eccf7 2024-11-17T08:15:48,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/6890b7afd8124947a2a8feba3307f071 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/6890b7afd8124947a2a8feba3307f071 2024-11-17T08:15:48,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/818e0eccbebd4b978e3584727e5aed1e to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/818e0eccbebd4b978e3584727e5aed1e 2024-11-17T08:15:48,213 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/2e781ec865144a6b893f9d42e0cd87b7 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/2e781ec865144a6b893f9d42e0cd87b7 2024-11-17T08:15:48,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/0c7330ebde3c432099230a3932394504 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/0c7330ebde3c432099230a3932394504 2024-11-17T08:15:48,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/84001bdf570d4fcd8f71211646f87260 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/84001bdf570d4fcd8f71211646f87260 2024-11-17T08:15:48,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/905c356fe62f44c8a4a9d80c967db163 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/905c356fe62f44c8a4a9d80c967db163 2024-11-17T08:15:48,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/0e985e982d66493cb376e07c3cf301b8 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/0e985e982d66493cb376e07c3cf301b8 2024-11-17T08:15:48,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/8486c769e35b4506ae6befabbb8827c2 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/8486c769e35b4506ae6befabbb8827c2 2024-11-17T08:15:48,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/1e85ad80bf764da0ba214a4e15adf5f3 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/1e85ad80bf764da0ba214a4e15adf5f3 2024-11-17T08:15:48,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/fbaeee0be0ba44ed92d1ee14a6c93394 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/fbaeee0be0ba44ed92d1ee14a6c93394 2024-11-17T08:15:48,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/0e99033d738541baaa4249d1865e41b8 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/0e99033d738541baaa4249d1865e41b8 2024-11-17T08:15:48,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/a4d0c85b8e2244d8b0f520895736d206 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/a4d0c85b8e2244d8b0f520895736d206 2024-11-17T08:15:48,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/579ac7af3e4f4d10834963770c34d21d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/579ac7af3e4f4d10834963770c34d21d 2024-11-17T08:15:48,225 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/8690451de16c4d1c95790bd167ffda20 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/8690451de16c4d1c95790bd167ffda20 2024-11-17T08:15:48,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/aec757bb230d4859a08ba36ee671eddc to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/aec757bb230d4859a08ba36ee671eddc 2024-11-17T08:15:48,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/dff2d328b3c647d1a4befe994e205807 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/dff2d328b3c647d1a4befe994e205807 2024-11-17T08:15:48,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/4569d78750764c1fa49b2e61a97399ce to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/4569d78750764c1fa49b2e61a97399ce 2024-11-17T08:15:48,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/a5c3ec4af12e4e19a7274b68b883f390 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/a5c3ec4af12e4e19a7274b68b883f390 2024-11-17T08:15:48,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/1f621d86623044388aa9a6e4b1f135d1 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/1f621d86623044388aa9a6e4b1f135d1 2024-11-17T08:15:48,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/d4f322dafeca45d39f569c64d589c043 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/d4f322dafeca45d39f569c64d589c043 2024-11-17T08:15:48,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/3d73084d14b9414eaa5204f13adb0c50 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/3d73084d14b9414eaa5204f13adb0c50 2024-11-17T08:15:48,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/d0bd8318e22449f3a03a0b65848567cd to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/d0bd8318e22449f3a03a0b65848567cd 2024-11-17T08:15:48,234 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/1c5ee9d7340944bfbdf9de419eb220ed to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/1c5ee9d7340944bfbdf9de419eb220ed 2024-11-17T08:15:48,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/3e2a1a973396414286f24c25eb3f75c4 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/3e2a1a973396414286f24c25eb3f75c4 2024-11-17T08:15:48,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/de660cc7282345ee9cc223895e7fdef0 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/de660cc7282345ee9cc223895e7fdef0 2024-11-17T08:15:48,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/a8b779504e7e47608bfef186329ffed8 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/a8b779504e7e47608bfef186329ffed8 2024-11-17T08:15:48,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/59914c4f2b1e4266972bb2e3da8a1892 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/59914c4f2b1e4266972bb2e3da8a1892 2024-11-17T08:15:48,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/a64edaeabb3c4cdf9717af4c1b456c88 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/a64edaeabb3c4cdf9717af4c1b456c88 2024-11-17T08:15:48,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/bd868fb6188f409fa30f13f467352b67 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/bd868fb6188f409fa30f13f467352b67 2024-11-17T08:15:48,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/42688fda3cd548159111b977f8b0ce55, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/cdaee554da47438b994bcc686641b0db, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/ca9043c695794e1481a66b8510898a43, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/0f3d1b57feee459da2f9ffd260c2277d, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/414dad6927d5444e86873fa316906adf, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/79e06c8ff51e4f5f850c94b9b620d138, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/781114373aa9499bbe8554b8f7c9d2ff, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/5b873549d2a54c4482e9e124b18ad440, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/3aa74db4167e463085e2f96da3d1bab9, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/b05e41b761c24f5097ce45a5bc696517, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/34ccb4809b7042b19b2b022b1a86fb56, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/d3b0ef541e24414facf46758adceb9b8, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/d609d251a1dc4ec490d4f211fa4f3031, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/4b06111c7bf74ceebbad6b51bfb9078b, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/8c92f2557a844838bc90c99258faa77b, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/fabc637b5dff4b2aaa53686b79f021b4, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/b50cc2d79234456c8640b3db69b4a699, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/13e8f2586e194b35956507bb09be41e3, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/8cfcc69e72d9408db28e5d629c9aa1d4, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/257df833c99c448fbbf7d3e7a298afd3, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/7f6f0c9890be4ec5847c680d3e273536, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/eac1a2b64d4a4405b64c2e84aec0a3dd, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/ce3aa03729594e22b6b7eef91aa5e7c4, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/ea557597fb7e4d96a812eac466f7e641, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/a565e8aa7fac4035bbb83a9033663f72, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/cbce5237275a414ca512f65ee28e291d, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/235373f8d06d4995bd05d5b5cbe7b53a, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/8589b505e79541f48cd3f225f84b6dc1, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/085573d53cdc4dc0bdd9880442e2300e, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/c83e41b1e56144e09fafc0c0f9b7c604] to archive 2024-11-17T08:15:48,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T08:15:48,244 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/42688fda3cd548159111b977f8b0ce55 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/42688fda3cd548159111b977f8b0ce55 2024-11-17T08:15:48,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/cdaee554da47438b994bcc686641b0db to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/cdaee554da47438b994bcc686641b0db 2024-11-17T08:15:48,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/ca9043c695794e1481a66b8510898a43 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/ca9043c695794e1481a66b8510898a43 2024-11-17T08:15:48,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/0f3d1b57feee459da2f9ffd260c2277d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/0f3d1b57feee459da2f9ffd260c2277d 2024-11-17T08:15:48,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/414dad6927d5444e86873fa316906adf to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/414dad6927d5444e86873fa316906adf 2024-11-17T08:15:48,251 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/79e06c8ff51e4f5f850c94b9b620d138 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/79e06c8ff51e4f5f850c94b9b620d138 2024-11-17T08:15:48,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/781114373aa9499bbe8554b8f7c9d2ff to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/781114373aa9499bbe8554b8f7c9d2ff 2024-11-17T08:15:48,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/5b873549d2a54c4482e9e124b18ad440 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/5b873549d2a54c4482e9e124b18ad440 2024-11-17T08:15:48,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/3aa74db4167e463085e2f96da3d1bab9 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/3aa74db4167e463085e2f96da3d1bab9 2024-11-17T08:15:48,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/b05e41b761c24f5097ce45a5bc696517 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/b05e41b761c24f5097ce45a5bc696517 2024-11-17T08:15:48,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/34ccb4809b7042b19b2b022b1a86fb56 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/34ccb4809b7042b19b2b022b1a86fb56 2024-11-17T08:15:48,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/d3b0ef541e24414facf46758adceb9b8 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/d3b0ef541e24414facf46758adceb9b8 2024-11-17T08:15:48,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/d609d251a1dc4ec490d4f211fa4f3031 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/d609d251a1dc4ec490d4f211fa4f3031 2024-11-17T08:15:48,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/4b06111c7bf74ceebbad6b51bfb9078b to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/4b06111c7bf74ceebbad6b51bfb9078b 2024-11-17T08:15:48,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/8c92f2557a844838bc90c99258faa77b to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/8c92f2557a844838bc90c99258faa77b 2024-11-17T08:15:48,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/fabc637b5dff4b2aaa53686b79f021b4 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/fabc637b5dff4b2aaa53686b79f021b4 2024-11-17T08:15:48,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/b50cc2d79234456c8640b3db69b4a699 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/b50cc2d79234456c8640b3db69b4a699 2024-11-17T08:15:48,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/13e8f2586e194b35956507bb09be41e3 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/13e8f2586e194b35956507bb09be41e3 2024-11-17T08:15:48,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/8cfcc69e72d9408db28e5d629c9aa1d4 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/8cfcc69e72d9408db28e5d629c9aa1d4 2024-11-17T08:15:48,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/257df833c99c448fbbf7d3e7a298afd3 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/257df833c99c448fbbf7d3e7a298afd3 2024-11-17T08:15:48,269 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/7f6f0c9890be4ec5847c680d3e273536 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/7f6f0c9890be4ec5847c680d3e273536 2024-11-17T08:15:48,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/eac1a2b64d4a4405b64c2e84aec0a3dd to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/eac1a2b64d4a4405b64c2e84aec0a3dd 2024-11-17T08:15:48,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/ce3aa03729594e22b6b7eef91aa5e7c4 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/ce3aa03729594e22b6b7eef91aa5e7c4 2024-11-17T08:15:48,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/ea557597fb7e4d96a812eac466f7e641 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/ea557597fb7e4d96a812eac466f7e641 2024-11-17T08:15:48,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/a565e8aa7fac4035bbb83a9033663f72 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/a565e8aa7fac4035bbb83a9033663f72 2024-11-17T08:15:48,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/cbce5237275a414ca512f65ee28e291d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/cbce5237275a414ca512f65ee28e291d 2024-11-17T08:15:48,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/235373f8d06d4995bd05d5b5cbe7b53a to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/235373f8d06d4995bd05d5b5cbe7b53a 2024-11-17T08:15:48,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/8589b505e79541f48cd3f225f84b6dc1 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/8589b505e79541f48cd3f225f84b6dc1 2024-11-17T08:15:48,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/085573d53cdc4dc0bdd9880442e2300e to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/085573d53cdc4dc0bdd9880442e2300e 2024-11-17T08:15:48,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/c83e41b1e56144e09fafc0c0f9b7c604 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/c83e41b1e56144e09fafc0c0f9b7c604 2024-11-17T08:15:48,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/0e06d9a598a745f0a7e175a008654198, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/78a3dbe261a34d94a7758e985f6bc1a7, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/a646f9f774df4b16a71bcba14a6746fe, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/59e5978a22824bdea6796897ddbce226, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/315dfcbc42484589b10f694ed68a847e, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/9d00d13b4ed247d79dbeb208975e2a34, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/e3b49af6f749459ebfbff1330a2f7ef6, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/8f7645d8e62848db9b1d71a5da45e07c, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/7d34eec3a9a8489391804b7c543c9324, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/2bf5beb49fcb4743b94d41c0e0e827a8, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/41cb4ad82f054c5baae27142e961264d, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/e52673007b7b4c01a09801081f08c76c, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/e662576b29cb48839081170c9aab3344, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/f15bc8b4c41c4d17ba3fdd8c41620303, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/a15d46d83b19472e857804275564b71a, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/b472e4d7f94c4a8dbd39a18cd1f566ad, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/d4f897ccf8264f43aa6e13fc47f5bf45, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/65c79b1932634d53a4c40c528cf842ef, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/aba476b22aa4484d9684c784602181a7, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/49d138e8c994496db2daffa36324468d, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/afad2a58aad842c781fade46f17fd2ed, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/12f651ca884b4286ae2ac5d0f8fb67c6, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/84eed278cec34f4a9860b2443d1b78a0, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/4d6dc4e7a6bc4bd28030dc6b1dc93ff0, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/87da5eeb83a3416b8f358d432a5bd1b2, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/4de51a14e6b041d5a85a0c8bdf743698, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/c6a13d53394d4120ac4c3952b2a931a7, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/b741a0c625c448608e37d1afa8a0ca5f, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/44ea2d3aa9cd43fd8e368deb5b992a2c, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/e2d4bd392c204c3487a38a3c0da6ab52] to archive 2024-11-17T08:15:48,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T08:15:48,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/0e06d9a598a745f0a7e175a008654198 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/0e06d9a598a745f0a7e175a008654198 2024-11-17T08:15:48,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/78a3dbe261a34d94a7758e985f6bc1a7 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/78a3dbe261a34d94a7758e985f6bc1a7 2024-11-17T08:15:48,285 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/a646f9f774df4b16a71bcba14a6746fe to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/a646f9f774df4b16a71bcba14a6746fe 2024-11-17T08:15:48,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/59e5978a22824bdea6796897ddbce226 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/59e5978a22824bdea6796897ddbce226 2024-11-17T08:15:48,287 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/315dfcbc42484589b10f694ed68a847e to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/315dfcbc42484589b10f694ed68a847e 2024-11-17T08:15:48,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/9d00d13b4ed247d79dbeb208975e2a34 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/9d00d13b4ed247d79dbeb208975e2a34 2024-11-17T08:15:48,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/e3b49af6f749459ebfbff1330a2f7ef6 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/e3b49af6f749459ebfbff1330a2f7ef6 2024-11-17T08:15:48,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/8f7645d8e62848db9b1d71a5da45e07c to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/8f7645d8e62848db9b1d71a5da45e07c 2024-11-17T08:15:48,290 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/7d34eec3a9a8489391804b7c543c9324 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/7d34eec3a9a8489391804b7c543c9324 2024-11-17T08:15:48,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/2bf5beb49fcb4743b94d41c0e0e827a8 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/2bf5beb49fcb4743b94d41c0e0e827a8 2024-11-17T08:15:48,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/41cb4ad82f054c5baae27142e961264d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/41cb4ad82f054c5baae27142e961264d 2024-11-17T08:15:48,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/e52673007b7b4c01a09801081f08c76c to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/e52673007b7b4c01a09801081f08c76c 2024-11-17T08:15:48,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/e662576b29cb48839081170c9aab3344 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/e662576b29cb48839081170c9aab3344 2024-11-17T08:15:48,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/f15bc8b4c41c4d17ba3fdd8c41620303 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/f15bc8b4c41c4d17ba3fdd8c41620303 2024-11-17T08:15:48,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/a15d46d83b19472e857804275564b71a to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/a15d46d83b19472e857804275564b71a 2024-11-17T08:15:48,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/b472e4d7f94c4a8dbd39a18cd1f566ad to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/b472e4d7f94c4a8dbd39a18cd1f566ad 2024-11-17T08:15:48,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/d4f897ccf8264f43aa6e13fc47f5bf45 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/d4f897ccf8264f43aa6e13fc47f5bf45 2024-11-17T08:15:48,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/65c79b1932634d53a4c40c528cf842ef to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/65c79b1932634d53a4c40c528cf842ef 2024-11-17T08:15:48,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/aba476b22aa4484d9684c784602181a7 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/aba476b22aa4484d9684c784602181a7 2024-11-17T08:15:48,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/49d138e8c994496db2daffa36324468d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/49d138e8c994496db2daffa36324468d 2024-11-17T08:15:48,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/afad2a58aad842c781fade46f17fd2ed to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/afad2a58aad842c781fade46f17fd2ed 2024-11-17T08:15:48,324 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/12f651ca884b4286ae2ac5d0f8fb67c6 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/12f651ca884b4286ae2ac5d0f8fb67c6 2024-11-17T08:15:48,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/84eed278cec34f4a9860b2443d1b78a0 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/84eed278cec34f4a9860b2443d1b78a0 2024-11-17T08:15:48,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/4d6dc4e7a6bc4bd28030dc6b1dc93ff0 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/4d6dc4e7a6bc4bd28030dc6b1dc93ff0 2024-11-17T08:15:48,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/87da5eeb83a3416b8f358d432a5bd1b2 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/87da5eeb83a3416b8f358d432a5bd1b2 2024-11-17T08:15:48,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/4de51a14e6b041d5a85a0c8bdf743698 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/4de51a14e6b041d5a85a0c8bdf743698 2024-11-17T08:15:48,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-17T08:15:48,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/c6a13d53394d4120ac4c3952b2a931a7 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/c6a13d53394d4120ac4c3952b2a931a7 2024-11-17T08:15:48,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/b741a0c625c448608e37d1afa8a0ca5f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/b741a0c625c448608e37d1afa8a0ca5f 2024-11-17T08:15:48,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/44ea2d3aa9cd43fd8e368deb5b992a2c to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/44ea2d3aa9cd43fd8e368deb5b992a2c 2024-11-17T08:15:48,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/e2d4bd392c204c3487a38a3c0da6ab52 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/e2d4bd392c204c3487a38a3c0da6ab52 2024-11-17T08:15:48,343 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/recovered.edits/422.seqid, newMaxSeqId=422, maxSeqId=1 2024-11-17T08:15:48,344 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. 2024-11-17T08:15:48,344 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for 23525a0a9ca7b911625b0315ffddfcb8: Waiting for close lock at 1731831347357Running coprocessor pre-close hooks at 1731831347357Disabling compacts and flushes for region at 1731831347357Disabling writes for close at 1731831347358 (+1 ms)Obtaining lock to block concurrent updates at 1731831347358Preparing flush snapshotting stores in 23525a0a9ca7b911625b0315ffddfcb8 at 1731831347358Finished memstore snapshotting TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1731831347358Flushing stores of TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8. at 1731831347359 (+1 ms)Flushing 23525a0a9ca7b911625b0315ffddfcb8/A: creating writer at 1731831347359Flushing 23525a0a9ca7b911625b0315ffddfcb8/A: appending metadata at 1731831347361 (+2 ms)Flushing 23525a0a9ca7b911625b0315ffddfcb8/A: closing flushed file at 1731831347361Flushing 23525a0a9ca7b911625b0315ffddfcb8/B: creating writer at 1731831347770 (+409 ms)Flushing 23525a0a9ca7b911625b0315ffddfcb8/B: appending metadata at 1731831347772 (+2 ms)Flushing 23525a0a9ca7b911625b0315ffddfcb8/B: closing flushed file at 1731831347772Flushing 23525a0a9ca7b911625b0315ffddfcb8/C: creating writer at 1731831347786 (+14 ms)Flushing 23525a0a9ca7b911625b0315ffddfcb8/C: appending metadata at 1731831347788 (+2 ms)Flushing 23525a0a9ca7b911625b0315ffddfcb8/C: closing flushed file at 1731831347788Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@716340e3: reopening flushed file at 1731831348196 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@262956b1: reopening flushed file at 1731831348199 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18816d9b: reopening flushed file at 1731831348203 (+4 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 23525a0a9ca7b911625b0315ffddfcb8 in 849ms, sequenceid=419, compaction requested=true at 1731831348207 (+4 ms)Writing region close event to WAL at 1731831348340 (+133 ms)Running coprocessor post-close hooks at 1731831348343 (+3 ms)Closed at 1731831348344 (+1 ms) 2024-11-17T08:15:48,345 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed 23525a0a9ca7b911625b0315ffddfcb8 2024-11-17T08:15:48,346 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=23525a0a9ca7b911625b0315ffddfcb8, regionState=CLOSED 2024-11-17T08:15:48,347 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 because future has completed 2024-11-17T08:15:48,350 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-17T08:15:48,350 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure 23525a0a9ca7b911625b0315ffddfcb8, server=e3d0adb87c01,40631,1731831247805 in 1.1430 sec 2024-11-17T08:15:48,351 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-17T08:15:48,351 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=23525a0a9ca7b911625b0315ffddfcb8, UNASSIGN in 1.1470 sec 2024-11-17T08:15:48,353 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-17T08:15:48,354 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1510 sec 2024-11-17T08:15:48,355 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731831348354"}]},"ts":"1731831348354"} 2024-11-17T08:15:48,356 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T08:15:48,356 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T08:15:48,357 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1630 sec 2024-11-17T08:15:49,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-17T08:15:49,344 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:15:49,345 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-17T08:15:49,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T08:15:49,347 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=92, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T08:15:49,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-17T08:15:49,348 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=92, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T08:15:49,352 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8 2024-11-17T08:15:49,356 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A, FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B, FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C, FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/recovered.edits] 2024-11-17T08:15:49,361 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/296088fc160a4421913eed3f08315b1e to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/296088fc160a4421913eed3f08315b1e 2024-11-17T08:15:49,363 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/4d0091bef6e341698aa09077aa473fe4 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/4d0091bef6e341698aa09077aa473fe4 2024-11-17T08:15:49,365 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/86dbeb0f0f23457bbd97e015a72c8457 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/A/86dbeb0f0f23457bbd97e015a72c8457 2024-11-17T08:15:49,368 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/07a2ff92f82b47ad8c4f65ccb810b1bb to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/07a2ff92f82b47ad8c4f65ccb810b1bb 2024-11-17T08:15:49,370 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/c8ce5eef9caa4ae099382596bdb8d2af to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/c8ce5eef9caa4ae099382596bdb8d2af 2024-11-17T08:15:49,371 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/cb6688a264a84b05b2df3ed81325774a to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/B/cb6688a264a84b05b2df3ed81325774a 2024-11-17T08:15:49,375 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/5d5f371216104e82996ce83a17a74aba to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/5d5f371216104e82996ce83a17a74aba 2024-11-17T08:15:49,376 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/603f3e7afd3f48fa88522a94fd88ba84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/603f3e7afd3f48fa88522a94fd88ba84 2024-11-17T08:15:49,377 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/6ef92cb51925426c83fe2cfd6df86b39 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/C/6ef92cb51925426c83fe2cfd6df86b39 2024-11-17T08:15:49,379 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/recovered.edits/422.seqid to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8/recovered.edits/422.seqid 2024-11-17T08:15:49,380 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/23525a0a9ca7b911625b0315ffddfcb8 2024-11-17T08:15:49,380 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T08:15:49,381 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=92, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T08:15:49,384 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T08:15:49,385 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T08:15:49,387 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=92, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T08:15:49,387 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T08:15:49,387 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731831349387"}]},"ts":"9223372036854775807"} 2024-11-17T08:15:49,388 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T08:15:49,388 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 23525a0a9ca7b911625b0315ffddfcb8, NAME => 'TestAcidGuarantees,,1731831316507.23525a0a9ca7b911625b0315ffddfcb8.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T08:15:49,388 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T08:15:49,388 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731831349388"}]},"ts":"9223372036854775807"} 2024-11-17T08:15:49,390 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T08:15:49,390 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=92, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T08:15:49,391 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 45 msec 2024-11-17T08:15:49,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-17T08:15:49,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T08:15:49,454 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:15:49,463 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=249 (was 251), OpenFileDescriptor=451 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=312 (was 278) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2410 (was 2170) - AvailableMemoryMB LEAK? - 2024-11-17T08:15:49,470 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=249, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=312, ProcessCount=11, AvailableMemoryMB=2410 2024-11-17T08:15:49,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T08:15:49,471 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T08:15:49,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T08:15:49,473 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T08:15:49,473 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:49,474 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 93 2024-11-17T08:15:49,474 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T08:15:49,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-17T08:15:49,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742149_1325 (size=1061) 2024-11-17T08:15:49,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-17T08:15:49,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-17T08:15:49,880 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85 2024-11-17T08:15:49,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742150_1326 (size=53) 2024-11-17T08:15:50,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-17T08:15:50,286 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T08:15:50,286 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b852432b87998249586edaadc388ac84, disabling compactions & flushes 2024-11-17T08:15:50,286 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:50,286 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:50,286 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. after waiting 0 ms 2024-11-17T08:15:50,286 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:50,286 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:50,286 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b852432b87998249586edaadc388ac84: Waiting for close lock at 1731831350286Disabling compacts and flushes for region at 1731831350286Disabling writes for close at 1731831350286Writing region close event to WAL at 1731831350286Closed at 1731831350286 2024-11-17T08:15:50,287 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T08:15:50,287 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731831350287"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731831350287"}]},"ts":"1731831350287"} 2024-11-17T08:15:50,289 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T08:15:50,290 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T08:15:50,290 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731831350290"}]},"ts":"1731831350290"} 2024-11-17T08:15:50,291 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T08:15:50,291 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b852432b87998249586edaadc388ac84, ASSIGN}] 2024-11-17T08:15:50,292 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b852432b87998249586edaadc388ac84, ASSIGN 2024-11-17T08:15:50,293 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b852432b87998249586edaadc388ac84, ASSIGN; state=OFFLINE, location=e3d0adb87c01,40631,1731831247805; forceNewPlan=false, retain=false 2024-11-17T08:15:50,443 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=b852432b87998249586edaadc388ac84, regionState=OPENING, regionLocation=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:50,445 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b852432b87998249586edaadc388ac84, ASSIGN because future has completed 2024-11-17T08:15:50,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805}] 2024-11-17T08:15:50,600 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:50,601 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7752): Opening region: {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} 2024-11-17T08:15:50,601 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b852432b87998249586edaadc388ac84 2024-11-17T08:15:50,601 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T08:15:50,601 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7794): checking encryption for b852432b87998249586edaadc388ac84 2024-11-17T08:15:50,601 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7797): checking classloading for b852432b87998249586edaadc388ac84 2024-11-17T08:15:50,603 INFO [StoreOpener-b852432b87998249586edaadc388ac84-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b852432b87998249586edaadc388ac84 2024-11-17T08:15:50,604 INFO [StoreOpener-b852432b87998249586edaadc388ac84-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T08:15:50,604 INFO [StoreOpener-b852432b87998249586edaadc388ac84-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b852432b87998249586edaadc388ac84 columnFamilyName A 2024-11-17T08:15:50,604 DEBUG [StoreOpener-b852432b87998249586edaadc388ac84-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:50,604 INFO [StoreOpener-b852432b87998249586edaadc388ac84-1 {}] regionserver.HStore(327): Store=b852432b87998249586edaadc388ac84/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T08:15:50,605 INFO [StoreOpener-b852432b87998249586edaadc388ac84-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b852432b87998249586edaadc388ac84 2024-11-17T08:15:50,606 INFO [StoreOpener-b852432b87998249586edaadc388ac84-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T08:15:50,606 INFO [StoreOpener-b852432b87998249586edaadc388ac84-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b852432b87998249586edaadc388ac84 columnFamilyName B 2024-11-17T08:15:50,606 DEBUG [StoreOpener-b852432b87998249586edaadc388ac84-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:50,606 INFO [StoreOpener-b852432b87998249586edaadc388ac84-1 {}] regionserver.HStore(327): Store=b852432b87998249586edaadc388ac84/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T08:15:50,606 INFO [StoreOpener-b852432b87998249586edaadc388ac84-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b852432b87998249586edaadc388ac84 2024-11-17T08:15:50,607 INFO [StoreOpener-b852432b87998249586edaadc388ac84-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T08:15:50,607 INFO [StoreOpener-b852432b87998249586edaadc388ac84-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b852432b87998249586edaadc388ac84 columnFamilyName C 2024-11-17T08:15:50,607 DEBUG [StoreOpener-b852432b87998249586edaadc388ac84-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:50,608 INFO [StoreOpener-b852432b87998249586edaadc388ac84-1 {}] regionserver.HStore(327): Store=b852432b87998249586edaadc388ac84/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T08:15:50,608 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:50,608 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1038): replaying wal for b852432b87998249586edaadc388ac84 2024-11-17T08:15:50,609 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84 2024-11-17T08:15:50,609 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84 2024-11-17T08:15:50,610 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1048): stopping wal replay for b852432b87998249586edaadc388ac84 2024-11-17T08:15:50,610 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1060): Cleaning up temporary data for b852432b87998249586edaadc388ac84 2024-11-17T08:15:50,610 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T08:15:50,611 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1093): writing seq id for b852432b87998249586edaadc388ac84 2024-11-17T08:15:50,614 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T08:15:50,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-17T08:15:50,614 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1114): Opened b852432b87998249586edaadc388ac84; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60155868, jitterRate=-0.10360771417617798}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T08:15:50,614 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b852432b87998249586edaadc388ac84 2024-11-17T08:15:50,615 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1006): Region open journal for b852432b87998249586edaadc388ac84: Running coprocessor pre-open hook at 1731831350601Writing region info on filesystem at 1731831350601Initializing all the Stores at 1731831350602 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731831350602Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731831350602Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731831350602Cleaning up temporary data from old regions at 1731831350610 (+8 ms)Running coprocessor post-open hooks at 1731831350614 (+4 ms)Region opened successfully at 1731831350615 (+1 ms) 2024-11-17T08:15:50,616 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., pid=95, masterSystemTime=1731831350597 2024-11-17T08:15:50,617 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:50,618 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:50,618 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=b852432b87998249586edaadc388ac84, regionState=OPEN, openSeqNum=2, regionLocation=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:50,620 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 because future has completed 2024-11-17T08:15:50,622 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-17T08:15:50,622 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 in 174 msec 2024-11-17T08:15:50,624 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-17T08:15:50,624 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b852432b87998249586edaadc388ac84, ASSIGN in 331 msec 2024-11-17T08:15:50,624 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T08:15:50,625 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731831350624"}]},"ts":"1731831350624"} 2024-11-17T08:15:50,626 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T08:15:50,627 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T08:15:50,629 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1560 sec 2024-11-17T08:15:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-17T08:15:51,624 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:15:51,626 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7befe57f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:51,627 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:15:51,627 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:15:51,629 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:15:51,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:15:51,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:15:51,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4281caa1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:51,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:15:51,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:15:51,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:51,632 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36964, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:15:51,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cf48933, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:51,634 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:15:51,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:15:51,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:15:51,637 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:15:51,637 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T08:15:51,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is e3d0adb87c01,45089,1731831246985 2024-11-17T08:15:51,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4a73ba3b 2024-11-17T08:15:51,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T08:15:51,639 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T08:15:51,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T08:15:51,641 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T08:15:51,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=96, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-17T08:15:51,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742151_1327 (size=1097) 2024-11-17T08:15:52,057 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-17T08:15:52,057 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-17T08:15:52,063 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T08:15:52,067 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b852432b87998249586edaadc388ac84, REOPEN/MOVE}] 2024-11-17T08:15:52,069 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b852432b87998249586edaadc388ac84, REOPEN/MOVE 2024-11-17T08:15:52,070 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=b852432b87998249586edaadc388ac84, regionState=CLOSING, regionLocation=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:52,072 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b852432b87998249586edaadc388ac84, REOPEN/MOVE because future has completed 2024-11-17T08:15:52,072 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T08:15:52,072 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805}] 2024-11-17T08:15:52,227 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(122): Close b852432b87998249586edaadc388ac84 2024-11-17T08:15:52,227 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T08:15:52,227 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1722): Closing b852432b87998249586edaadc388ac84, disabling compactions & flushes 2024-11-17T08:15:52,227 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:52,228 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:52,228 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. after waiting 0 ms 2024-11-17T08:15:52,228 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:52,238 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T08:15:52,239 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:52,239 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1676): Region close journal for b852432b87998249586edaadc388ac84: Waiting for close lock at 1731831352227Running coprocessor pre-close hooks at 1731831352227Disabling compacts and flushes for region at 1731831352227Disabling writes for close at 1731831352228 (+1 ms)Writing region close event to WAL at 1731831352229 (+1 ms)Running coprocessor post-close hooks at 1731831352239 (+10 ms)Closed at 1731831352239 2024-11-17T08:15:52,239 WARN [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegionServer(3299): Not adding moved region record: b852432b87998249586edaadc388ac84 to self. 2024-11-17T08:15:52,241 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(157): Closed b852432b87998249586edaadc388ac84 2024-11-17T08:15:52,242 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=b852432b87998249586edaadc388ac84, regionState=CLOSED 2024-11-17T08:15:52,242 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45089 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=CLOSED, location=null, table=TestAcidGuarantees, region=b852432b87998249586edaadc388ac84. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-17T08:15:52,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 because future has completed 2024-11-17T08:15:52,246 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-11-17T08:15:52,246 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; CloseRegionProcedure b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 in 172 msec 2024-11-17T08:15:52,246 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b852432b87998249586edaadc388ac84, REOPEN/MOVE; state=CLOSED, location=e3d0adb87c01,40631,1731831247805; forceNewPlan=false, retain=true 2024-11-17T08:15:52,397 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=b852432b87998249586edaadc388ac84, regionState=OPENING, regionLocation=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:52,398 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b852432b87998249586edaadc388ac84, REOPEN/MOVE because future has completed 2024-11-17T08:15:52,398 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805}] 2024-11-17T08:15:52,553 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:52,553 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7752): Opening region: {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} 2024-11-17T08:15:52,553 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b852432b87998249586edaadc388ac84 2024-11-17T08:15:52,553 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T08:15:52,553 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7794): checking encryption for b852432b87998249586edaadc388ac84 2024-11-17T08:15:52,553 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7797): checking classloading for b852432b87998249586edaadc388ac84 2024-11-17T08:15:52,555 INFO [StoreOpener-b852432b87998249586edaadc388ac84-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b852432b87998249586edaadc388ac84 2024-11-17T08:15:52,555 INFO [StoreOpener-b852432b87998249586edaadc388ac84-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T08:15:52,555 INFO [StoreOpener-b852432b87998249586edaadc388ac84-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b852432b87998249586edaadc388ac84 columnFamilyName A 2024-11-17T08:15:52,556 DEBUG [StoreOpener-b852432b87998249586edaadc388ac84-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:52,557 INFO [StoreOpener-b852432b87998249586edaadc388ac84-1 {}] regionserver.HStore(327): Store=b852432b87998249586edaadc388ac84/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T08:15:52,557 INFO [StoreOpener-b852432b87998249586edaadc388ac84-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b852432b87998249586edaadc388ac84 2024-11-17T08:15:52,558 INFO [StoreOpener-b852432b87998249586edaadc388ac84-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T08:15:52,558 INFO [StoreOpener-b852432b87998249586edaadc388ac84-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b852432b87998249586edaadc388ac84 columnFamilyName B 2024-11-17T08:15:52,558 DEBUG [StoreOpener-b852432b87998249586edaadc388ac84-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:52,558 INFO [StoreOpener-b852432b87998249586edaadc388ac84-1 {}] regionserver.HStore(327): Store=b852432b87998249586edaadc388ac84/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T08:15:52,558 INFO [StoreOpener-b852432b87998249586edaadc388ac84-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b852432b87998249586edaadc388ac84 2024-11-17T08:15:52,559 INFO [StoreOpener-b852432b87998249586edaadc388ac84-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T08:15:52,559 INFO [StoreOpener-b852432b87998249586edaadc388ac84-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b852432b87998249586edaadc388ac84 columnFamilyName C 2024-11-17T08:15:52,559 DEBUG [StoreOpener-b852432b87998249586edaadc388ac84-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:52,559 INFO [StoreOpener-b852432b87998249586edaadc388ac84-1 {}] regionserver.HStore(327): Store=b852432b87998249586edaadc388ac84/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T08:15:52,559 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:52,559 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1038): replaying wal for b852432b87998249586edaadc388ac84 2024-11-17T08:15:52,560 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84 2024-11-17T08:15:52,561 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84 2024-11-17T08:15:52,561 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1048): stopping wal replay for b852432b87998249586edaadc388ac84 2024-11-17T08:15:52,562 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1060): Cleaning up temporary data for b852432b87998249586edaadc388ac84 2024-11-17T08:15:52,562 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T08:15:52,563 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1093): writing seq id for b852432b87998249586edaadc388ac84 2024-11-17T08:15:52,564 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1114): Opened b852432b87998249586edaadc388ac84; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74500840, jitterRate=0.11014902591705322}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T08:15:52,564 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b852432b87998249586edaadc388ac84 2024-11-17T08:15:52,564 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1006): Region open journal for b852432b87998249586edaadc388ac84: Running coprocessor pre-open hook at 1731831352553Writing region info on filesystem at 1731831352553Initializing all the Stores at 1731831352554 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731831352554Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731831352554Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731831352554Cleaning up temporary data from old regions at 1731831352562 (+8 ms)Running coprocessor post-open hooks at 1731831352564 (+2 ms)Region opened successfully at 1731831352564 2024-11-17T08:15:52,565 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., pid=100, masterSystemTime=1731831352550 2024-11-17T08:15:52,566 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:52,566 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:52,566 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=b852432b87998249586edaadc388ac84, regionState=OPEN, openSeqNum=5, regionLocation=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:52,567 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 because future has completed 2024-11-17T08:15:52,569 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=98 2024-11-17T08:15:52,569 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=98, state=SUCCESS, hasLock=false; OpenRegionProcedure b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 in 170 msec 2024-11-17T08:15:52,571 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-17T08:15:52,571 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b852432b87998249586edaadc388ac84, REOPEN/MOVE in 502 msec 2024-11-17T08:15:52,572 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-17T08:15:52,572 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 509 msec 2024-11-17T08:15:52,575 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 932 msec 2024-11-17T08:15:52,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-17T08:15:52,575 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:15:52,576 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b10b9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:52,576 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:15:52,576 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:15:52,577 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:15:52,577 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:15:52,577 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:15:52,577 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69dde57b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:52,577 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:15:52,577 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:15:52,577 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:52,578 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49592, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:15:52,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ee4cd1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:52,579 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c265f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:52,579 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:15:52,579 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:15:52,579 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:15:52,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:15:52,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:15:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7efc67a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:15:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:15:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:52,580 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49604, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:15:52,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ef9b3dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:52,581 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c21b87a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:52,581 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:15:52,582 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:15:52,582 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:15:52,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:15:52,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:15:52,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10dbd998, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:52,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:15:52,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:15:52,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:52,584 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:15:52,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a8362cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:52,585 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35e136ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:52,585 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:15:52,585 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:15:52,585 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:15:52,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:15:52,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:15:52,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dcecc57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:52,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:15:52,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:15:52,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:52,586 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49636, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:15:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d3dd0d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:52,588 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f004f7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:52,588 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:15:52,588 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:15:52,588 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:15:52,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:15:52,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:15:52,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@368eec50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:52,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:15:52,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:15:52,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:52,590 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:15:52,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f34d9b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:52,591 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35aa2c8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:52,591 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:15:52,591 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:15:52,592 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:15:52,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:15:52,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:15:52,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2da392bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:52,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:15:52,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:15:52,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:52,593 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49670, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:15:52,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a2e299c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:52,594 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53f9edfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:52,594 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:15:52,594 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:15:52,595 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:15:52,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:15:52,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:15:52,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12621dc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:52,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:15:52,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:15:52,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:52,596 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:15:52,596 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fc5b81b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:52,597 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43065a6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:52,597 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:15:52,597 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:15:52,598 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:15:52,598 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:15:52,598 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:15:52,598 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@799b5c02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:52,598 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:15:52,598 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:15:52,598 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:52,599 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:15:52,599 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ba30e33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:52,600 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24bcd92c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:52,600 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:15:52,600 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:15:52,601 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:15:52,601 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:15:52,601 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:15:52,601 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d8e7032, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:52,601 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:15:52,601 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:15:52,601 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:52,602 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49716, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:15:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bc89b14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:52,603 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b779047, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:52,603 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:15:52,603 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:15:52,604 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:15:52,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:15:52,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:15:52,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@264fc596, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:52,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:15:52,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:15:52,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:15:52,605 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:15:52,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6081c405, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:15:52,606 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:15:52,607 DEBUG [Thread-1516 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:15:52,607 DEBUG [Thread-1503 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:15:52,607 DEBUG [Thread-1510 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:15:52,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-17T08:15:52,607 DEBUG [Thread-1518 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:15:52,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:15:52,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:15:52,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:15:52,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:15:52,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:15:52,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:15:52,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T08:15:52,608 DEBUG [Thread-1501 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:15:52,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:15:52,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:15:52,608 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:15:52,609 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:15:52,609 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:15:52,609 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:15:52,609 DEBUG [Thread-1499 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:15:52,609 DEBUG [Thread-1512 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:15:52,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:15:52,610 DEBUG [Thread-1514 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:15:52,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:15:52,610 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53450, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:15:52,610 DEBUG [Thread-1505 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:15:52,610 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:15:52,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:15:52,610 DEBUG [Thread-1507 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:15:52,611 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:15:52,611 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:15:52,611 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:15:52,611 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:15:52,611 DEBUG [Thread-1512 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:15:52,611 DEBUG [Thread-1512 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:15:52,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:15:52,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:15:52,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:15:52,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:15:52,612 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53500, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:15:52,612 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:15:52,612 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:15:52,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5] 2024-11-17T08:15:52,612 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:15:52,612 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:15:52,613 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:15:52,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5] 2024-11-17T08:15:52,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5] 2024-11-17T08:15:52,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5] 2024-11-17T08:15:52,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5] 2024-11-17T08:15:52,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5] 2024-11-17T08:15:52,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5] 2024-11-17T08:15:52,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5] 2024-11-17T08:15:52,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5] 2024-11-17T08:15:52,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5] 2024-11-17T08:15:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on b852432b87998249586edaadc388ac84 2024-11-17T08:15:52,625 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b852432b87998249586edaadc388ac84 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T08:15:52,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=A 2024-11-17T08:15:52,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:52,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=B 2024-11-17T08:15:52,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:52,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=C 2024-11-17T08:15:52,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:52,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411175c4cf5508fb94846a3ee13fc6d7c763c_b852432b87998249586edaadc388ac84 is 50, key is test_row_0/A:col10/1731831352619/Put/seqid=0 2024-11-17T08:15:52,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742152_1328 (size=14594) 2024-11-17T08:15:52,654 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:52,658 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411175c4cf5508fb94846a3ee13fc6d7c763c_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175c4cf5508fb94846a3ee13fc6d7c763c_b852432b87998249586edaadc388ac84 2024-11-17T08:15:52,659 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/b7545be14c1a432f83eeb367c7f7eb9c, store: [table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:15:52,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/b7545be14c1a432f83eeb367c7f7eb9c is 175, key is test_row_0/A:col10/1731831352619/Put/seqid=0 2024-11-17T08:15:52,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742153_1329 (size=39545) 2024-11-17T08:15:52,664 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/b7545be14c1a432f83eeb367c7f7eb9c 2024-11-17T08:15:52,669 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731831352642, max timestamp=1731831352664Num uniques 859; 2024-11-17T08:15:52,669 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-17T08:15:52,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731831352642, max timestamp=1731831352666Num uniques 859; 2024-11-17T08:15:52,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-17T08:15:52,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731831352642, max timestamp=1731831352666Num uniques 859; 2024-11-17T08:15:52,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-17T08:15:52,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/d37e6908cb3f4f0a96e3608dca56ed45 is 50, key is test_row_0/B:col10/1731831352619/Put/seqid=0 2024-11-17T08:15:52,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742154_1330 (size=9657) 2024-11-17T08:15:52,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/d37e6908cb3f4f0a96e3608dca56ed45 2024-11-17T08:15:52,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:52,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T08:15:52,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831412705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:52,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:52,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:52,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:52,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:52,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831412706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:52,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:52,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:52,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:52,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:52,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831412709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:52,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:52,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:52,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:52,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53496 deadline: 1731831412710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:52,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:52,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:52,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831412712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:52,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/9a33c619f6a44d20bdf8a75cff1338a5 is 50, key is test_row_0/C:col10/1731831352619/Put/seqid=0 2024-11-17T08:15:52,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:52,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:52,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:52,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742155_1331 (size=9657) 2024-11-17T08:15:52,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/9a33c619f6a44d20bdf8a75cff1338a5 2024-11-17T08:15:52,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/b7545be14c1a432f83eeb367c7f7eb9c as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/b7545be14c1a432f83eeb367c7f7eb9c 2024-11-17T08:15:52,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/b7545be14c1a432f83eeb367c7f7eb9c, entries=200, sequenceid=14, filesize=38.6 K 2024-11-17T08:15:52,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/d37e6908cb3f4f0a96e3608dca56ed45 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/d37e6908cb3f4f0a96e3608dca56ed45 2024-11-17T08:15:52,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-17T08:15:52,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:52,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:15:52,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:52,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:52,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:52,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:52,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/d37e6908cb3f4f0a96e3608dca56ed45, entries=100, sequenceid=14, filesize=9.4 K 2024-11-17T08:15:52,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/9a33c619f6a44d20bdf8a75cff1338a5 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/9a33c619f6a44d20bdf8a75cff1338a5 2024-11-17T08:15:52,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/9a33c619f6a44d20bdf8a75cff1338a5, entries=100, sequenceid=14, filesize=9.4 K 2024-11-17T08:15:52,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.42 KB/139690 for b852432b87998249586edaadc388ac84 in 166ms, sequenceid=14, compaction requested=false 2024-11-17T08:15:52,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b852432b87998249586edaadc388ac84: 2024-11-17T08:15:52,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on b852432b87998249586edaadc388ac84 2024-11-17T08:15:52,841 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b852432b87998249586edaadc388ac84 3/3 column families, dataSize=143.13 KB heapSize=395.92 KB 2024-11-17T08:15:52,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=A 2024-11-17T08:15:52,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:15:52,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=B 2024-11-17T08:15:52,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:15:52,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=C 2024-11-17T08:15:52,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:15:52,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117edc8f131fd99499189a48485b9032bb4_b852432b87998249586edaadc388ac84 is 50, key is test_row_0/A:col10/1731831352836/Put/seqid=0 2024-11-17T08:15:52,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742156_1332 (size=14594) 2024-11-17T08:15:52,861 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:52,864 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117edc8f131fd99499189a48485b9032bb4_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117edc8f131fd99499189a48485b9032bb4_b852432b87998249586edaadc388ac84 2024-11-17T08:15:52,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:52,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:52,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831412847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:52,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831412848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:52,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:52,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:52,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:52,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:52,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53496 deadline: 1731831412849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:52,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:52,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:52,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:52,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:52,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:52,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:52,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:52,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831412856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:52,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:52,866 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/ffb3418ecacf41ceae841c4e77d1e06f, store: [table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:15:52,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:52,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:52,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/ffb3418ecacf41ceae841c4e77d1e06f is 175, key is test_row_0/A:col10/1731831352836/Put/seqid=0 2024-11-17T08:15:52,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:52,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831412866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:52,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:52,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:52,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:52,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742157_1333 (size=39549) 2024-11-17T08:15:52,875 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/ffb3418ecacf41ceae841c4e77d1e06f 2024-11-17T08:15:52,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/0e0afce9248c4c679dabb0d639d5bd7c is 50, key is test_row_0/B:col10/1731831352836/Put/seqid=0 2024-11-17T08:15:52,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742158_1334 (size=12001) 2024-11-17T08:15:52,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-17T08:15:52,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:52,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:15:52,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:52,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:52,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:52,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:52,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T08:15:52,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:52,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831412974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:52,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:52,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:52,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831412974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:52,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:52,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:52,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:52,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:52,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:52,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:52,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831412974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:52,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:52,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:52,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53496 deadline: 1731831412974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:52,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:52,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:52,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:52,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:52,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831412984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:52,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:52,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:52,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:52,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:52,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:53,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-17T08:15:53,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:53,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:15:53,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:53,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:53,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:53,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:53,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:53,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831413194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:53,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:53,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:53,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:53,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:53,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831413194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:53,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:53,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:53,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:53,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831413194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:53,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:53,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:53,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:53,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:53,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:53,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53496 deadline: 1731831413194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:53,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:53,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:53,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:53,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:53,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831413194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:53,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:53,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:53,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:53,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-17T08:15:53,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:53,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:15:53,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:53,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:53,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:53,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:53,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T08:15:53,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/0e0afce9248c4c679dabb0d639d5bd7c 2024-11-17T08:15:53,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/d4078f8657454a7baf481a8fcf606cd9 is 50, key is test_row_0/C:col10/1731831352836/Put/seqid=0 2024-11-17T08:15:53,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742159_1335 (size=12001) 2024-11-17T08:15:53,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-17T08:15:53,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:53,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:15:53,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:53,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:53,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:53,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:53,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:53,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831413504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:53,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:53,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:53,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:53,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831413504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:53,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:53,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:53,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831413504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:53,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:53,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:53,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:53,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:53,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:53,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:53,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:53,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53496 deadline: 1731831413513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:53,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:53,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:53,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:53,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:53,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831413513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:53,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:53,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:53,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:53,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-17T08:15:53,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:53,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:15:53,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:53,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:53,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:53,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:53,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-17T08:15:53,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:53,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:15:53,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:53,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:53,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:53,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:53,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/d4078f8657454a7baf481a8fcf606cd9 2024-11-17T08:15:53,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/ffb3418ecacf41ceae841c4e77d1e06f as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/ffb3418ecacf41ceae841c4e77d1e06f 2024-11-17T08:15:53,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/ffb3418ecacf41ceae841c4e77d1e06f, entries=200, sequenceid=49, filesize=38.6 K 2024-11-17T08:15:53,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/0e0afce9248c4c679dabb0d639d5bd7c as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/0e0afce9248c4c679dabb0d639d5bd7c 2024-11-17T08:15:53,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/0e0afce9248c4c679dabb0d639d5bd7c, entries=150, sequenceid=49, filesize=11.7 K 2024-11-17T08:15:53,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/d4078f8657454a7baf481a8fcf606cd9 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/d4078f8657454a7baf481a8fcf606cd9 2024-11-17T08:15:53,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/d4078f8657454a7baf481a8fcf606cd9, entries=150, sequenceid=49, filesize=11.7 K 2024-11-17T08:15:53,739 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~143.13 KB/146560, heapSize ~395.88 KB/405376, currentSize=40.25 KB/41220 for b852432b87998249586edaadc388ac84 in 898ms, sequenceid=49, compaction requested=false 2024-11-17T08:15:53,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b852432b87998249586edaadc388ac84: 2024-11-17T08:15:53,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T08:15:53,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-17T08:15:53,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:53,840 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing b852432b87998249586edaadc388ac84 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-17T08:15:53,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=A 2024-11-17T08:15:53,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:53,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=B 2024-11-17T08:15:53,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:53,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=C 2024-11-17T08:15:53,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:53,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111750f61e66d4884e6cbf5a3da1bcb50511_b852432b87998249586edaadc388ac84 is 50, key is test_row_0/A:col10/1731831352845/Put/seqid=0 2024-11-17T08:15:53,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742160_1336 (size=12154) 2024-11-17T08:15:53,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:53,881 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T08:15:53,885 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111750f61e66d4884e6cbf5a3da1bcb50511_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111750f61e66d4884e6cbf5a3da1bcb50511_b852432b87998249586edaadc388ac84 2024-11-17T08:15:53,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/0ba8e8a75621419fa50eb91f5da4edd7, store: [table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:15:53,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/0ba8e8a75621419fa50eb91f5da4edd7 is 175, key is test_row_0/A:col10/1731831352845/Put/seqid=0 2024-11-17T08:15:53,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742161_1337 (size=30955) 2024-11-17T08:15:54,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on b852432b87998249586edaadc388ac84 2024-11-17T08:15:54,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:15:54,075 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731831354047, max timestamp=1731831354072Num uniques 859; 2024-11-17T08:15:54,075 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T08:15:54,075 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731831354047, max timestamp=1731831354072Num uniques 859; 2024-11-17T08:15:54,075 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731831354047, max timestamp=1731831354072Num uniques 859; 2024-11-17T08:15:54,075 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T08:15:54,075 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T08:15:54,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:54,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53496 deadline: 1731831414115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:54,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:54,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:54,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:54,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:54,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831414116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:54,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:54,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:54,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:54,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:54,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831414119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:54,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:54,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:54,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:54,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:54,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831414133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:54,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:54,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:54,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831414133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:54,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:54,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:54,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:54,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:54,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:54,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:54,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831414247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:54,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:54,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53496 deadline: 1731831414248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:54,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:54,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:54,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:54,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:54,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831414248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:54,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:54,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:54,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:54,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:54,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:54,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:54,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831414254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:54,268 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:54,268 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:54,268 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:54,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:54,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831414264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:54,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:54,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:54,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:54,332 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=58, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/0ba8e8a75621419fa50eb91f5da4edd7 2024-11-17T08:15:54,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/ceeae6fb0a524d978b5e762a5ad33a4a is 50, key is test_row_0/B:col10/1731831352845/Put/seqid=0 2024-11-17T08:15:54,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742162_1338 (size=12001) 2024-11-17T08:15:54,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:54,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53496 deadline: 1731831414455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:54,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:54,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831414455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:54,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:54,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:54,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:54,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:54,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:54,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:54,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:54,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831414463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:54,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:54,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:54,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:54,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:54,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831414474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:54,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:54,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:54,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:54,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:54,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831414484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:54,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:54,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:54,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:54,742 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/ceeae6fb0a524d978b5e762a5ad33a4a 2024-11-17T08:15:54,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/2341bb1c27574b838b5c811ebca77caa is 50, key is test_row_0/C:col10/1731831352845/Put/seqid=0 2024-11-17T08:15:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T08:15:54,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742163_1339 (size=12001) 2024-11-17T08:15:54,760 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/2341bb1c27574b838b5c811ebca77caa 2024-11-17T08:15:54,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/0ba8e8a75621419fa50eb91f5da4edd7 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/0ba8e8a75621419fa50eb91f5da4edd7 2024-11-17T08:15:54,772 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/0ba8e8a75621419fa50eb91f5da4edd7, entries=150, sequenceid=58, filesize=30.2 K 2024-11-17T08:15:54,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/ceeae6fb0a524d978b5e762a5ad33a4a as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/ceeae6fb0a524d978b5e762a5ad33a4a 2024-11-17T08:15:54,781 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/ceeae6fb0a524d978b5e762a5ad33a4a, entries=150, sequenceid=58, filesize=11.7 K 2024-11-17T08:15:54,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/2341bb1c27574b838b5c811ebca77caa as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/2341bb1c27574b838b5c811ebca77caa 2024-11-17T08:15:54,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831414775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:54,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:54,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:54,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:54,788 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/2341bb1c27574b838b5c811ebca77caa, entries=150, sequenceid=58, filesize=11.7 K 2024-11-17T08:15:54,789 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=147.60 KB/151140 for b852432b87998249586edaadc388ac84 in 949ms, sequenceid=58, compaction requested=true 2024-11-17T08:15:54,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for b852432b87998249586edaadc388ac84: 2024-11-17T08:15:54,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:54,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-17T08:15:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-17T08:15:54,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-17T08:15:54,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1800 sec 2024-11-17T08:15:54,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on b852432b87998249586edaadc388ac84 2024-11-17T08:15:54,796 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.1880 sec 2024-11-17T08:15:54,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731831354114, max timestamp=1731831354793Num uniques 1118; 2024-11-17T08:15:54,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-17T08:15:54,798 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b852432b87998249586edaadc388ac84 3/3 column families, dataSize=111.82 KB heapSize=312.52 KB 2024-11-17T08:15:54,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731831354114, max timestamp=1731831354793Num uniques 1118; 2024-11-17T08:15:54,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731831354114, max timestamp=1731831354793Num uniques 1118; 2024-11-17T08:15:54,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-17T08:15:54,798 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-17T08:15:54,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=A 2024-11-17T08:15:54,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:15:54,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=B 2024-11-17T08:15:54,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:15:54,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=C 2024-11-17T08:15:54,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:15:54,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117931077cf99a346fb9ee15872c71c7d78_b852432b87998249586edaadc388ac84 is 50, key is test_row_0/A:col10/1731831354793/Put/seqid=0 2024-11-17T08:15:54,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742164_1340 (size=17034) 2024-11-17T08:15:54,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1731831354840, max timestamp=1731831354858Num uniques 859; 2024-11-17T08:15:54,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1731831354840, max timestamp=1731831354858Num uniques 859; 2024-11-17T08:15:54,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-17T08:15:54,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-17T08:15:54,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1731831354840, max timestamp=1731831354858Num uniques 859; 2024-11-17T08:15:54,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-17T08:15:54,928 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731831354903, max timestamp=1731831354924Num uniques 1168; 2024-11-17T08:15:54,928 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=9978, suffix heap size=149440,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=218 2024-11-17T08:15:54,928 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731831354903, max timestamp=1731831354924Num uniques 1168; 2024-11-17T08:15:54,928 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=9978, suffix heap size=149440,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=218 2024-11-17T08:15:54,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731831354903, max timestamp=1731831354924Num uniques 1168; 2024-11-17T08:15:54,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=9978, suffix heap size=149440,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=218 2024-11-17T08:15:55,015 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731831354965, max timestamp=1731831355001Num uniques 1077; 2024-11-17T08:15:55,016 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-17T08:15:55,016 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731831354965, max timestamp=1731831355001Num uniques 1077; 2024-11-17T08:15:55,016 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-17T08:15:55,016 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731831354967, max timestamp=1731831355001Num uniques 1077; 2024-11-17T08:15:55,016 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-17T08:15:55,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731831355056, max timestamp=1731831355080Num uniques 1136; 2024-11-17T08:15:55,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731831355056, max timestamp=1731831355080Num uniques 1136; 2024-11-17T08:15:55,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=10806, suffix heap size=146112,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=236 2024-11-17T08:15:55,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731831355056, max timestamp=1731831355080Num uniques 1036; 2024-11-17T08:15:55,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=10806, suffix heap size=146112,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=236 2024-11-17T08:15:55,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-17T08:15:55,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1731831355121, max timestamp=1731831355154Num uniques 1095; 2024-11-17T08:15:55,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=15800, suffix heap size=141848,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=345 2024-11-17T08:15:55,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1731831355121, max timestamp=1731831355154Num uniques 1095; 2024-11-17T08:15:55,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=15800, suffix heap size=141848,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=345 2024-11-17T08:15:55,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1731831355121, max timestamp=1731831355154Num uniques 1095; 2024-11-17T08:15:55,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=15800, suffix heap size=141848,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=345 2024-11-17T08:15:55,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:55,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831415200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:55,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:55,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:55,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:55,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:55,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831415201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:55,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:55,215 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:55,215 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:55,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831415204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:55,215 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:55,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:55,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:55,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:55,216 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:55,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:55,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53496 deadline: 1731831415209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:55,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:55,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:55,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:55,220 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117931077cf99a346fb9ee15872c71c7d78_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117931077cf99a346fb9ee15872c71c7d78_b852432b87998249586edaadc388ac84 2024-11-17T08:15:55,220 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/86132cf5fbc949b1b31513a745152274, store: [table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:15:55,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/86132cf5fbc949b1b31513a745152274 is 175, key is test_row_0/A:col10/1731831354793/Put/seqid=0 2024-11-17T08:15:55,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742165_1341 (size=48139) 2024-11-17T08:15:55,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:55,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831415294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:55,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:55,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:55,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:55,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:55,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831415313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:55,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:55,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:55,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:55,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:55,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831415323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:55,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:55,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831415323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:55,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:55,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:55,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:55,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:55,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:55,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:55,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53496 deadline: 1731831415323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:55,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:55,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:55,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:55,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:55,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:55,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831415533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:55,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:55,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:55,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:55,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831415533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:55,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:55,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:55,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:55,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831415544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:55,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53496 deadline: 1731831415544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:55,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:55,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:55,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:55,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:55,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:55,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:55,624 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/86132cf5fbc949b1b31513a745152274 2024-11-17T08:15:55,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/4a3c8d88746e4e5682ed23bede52e0b8 is 50, key is test_row_0/B:col10/1731831354793/Put/seqid=0 2024-11-17T08:15:55,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742166_1342 (size=12001) 2024-11-17T08:15:55,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/4a3c8d88746e4e5682ed23bede52e0b8 2024-11-17T08:15:55,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/2d03be7081de4ee3a282d7070b4a3938 is 50, key is test_row_0/C:col10/1731831354793/Put/seqid=0 2024-11-17T08:15:55,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742167_1343 (size=12001) 2024-11-17T08:15:55,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831415854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:55,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:55,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:55,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:55,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831415854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:55,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:55,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:55,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:55,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53496 deadline: 1731831415854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:55,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:55,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:55,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:55,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:55,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831415864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:55,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:55,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:55,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:56,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/2d03be7081de4ee3a282d7070b4a3938 2024-11-17T08:15:56,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/86132cf5fbc949b1b31513a745152274 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/86132cf5fbc949b1b31513a745152274 2024-11-17T08:15:56,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/86132cf5fbc949b1b31513a745152274, entries=250, sequenceid=97, filesize=47.0 K 2024-11-17T08:15:56,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/4a3c8d88746e4e5682ed23bede52e0b8 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/4a3c8d88746e4e5682ed23bede52e0b8 2024-11-17T08:15:56,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/4a3c8d88746e4e5682ed23bede52e0b8, entries=150, sequenceid=97, filesize=11.7 K 2024-11-17T08:15:56,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/2d03be7081de4ee3a282d7070b4a3938 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/2d03be7081de4ee3a282d7070b4a3938 2024-11-17T08:15:56,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/2d03be7081de4ee3a282d7070b4a3938, entries=150, sequenceid=97, filesize=11.7 K 2024-11-17T08:15:56,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~85.27 KB/87312, currentSize=154.31 KB/158010 for b852432b87998249586edaadc388ac84 in 1425ms, sequenceid=97, compaction requested=true 2024-11-17T08:15:56,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b852432b87998249586edaadc388ac84: 2024-11-17T08:15:56,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b852432b87998249586edaadc388ac84:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:15:56,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:56,223 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T08:15:56,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b852432b87998249586edaadc388ac84:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:15:56,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:56,223 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T08:15:56,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b852432b87998249586edaadc388ac84:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:15:56,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:15:56,224 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 158188 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T08:15:56,224 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T08:15:56,224 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): b852432b87998249586edaadc388ac84/B is initiating minor compaction (all files) 2024-11-17T08:15:56,224 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): b852432b87998249586edaadc388ac84/A is initiating minor compaction (all files) 2024-11-17T08:15:56,225 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b852432b87998249586edaadc388ac84/B in TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:56,225 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b852432b87998249586edaadc388ac84/A in TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:56,225 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/b7545be14c1a432f83eeb367c7f7eb9c, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/ffb3418ecacf41ceae841c4e77d1e06f, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/0ba8e8a75621419fa50eb91f5da4edd7, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/86132cf5fbc949b1b31513a745152274] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp, totalSize=154.5 K 2024-11-17T08:15:56,225 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/d37e6908cb3f4f0a96e3608dca56ed45, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/0e0afce9248c4c679dabb0d639d5bd7c, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/ceeae6fb0a524d978b5e762a5ad33a4a, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/4a3c8d88746e4e5682ed23bede52e0b8] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp, totalSize=44.6 K 2024-11-17T08:15:56,225 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:56,225 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. files: [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/b7545be14c1a432f83eeb367c7f7eb9c, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/ffb3418ecacf41ceae841c4e77d1e06f, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/0ba8e8a75621419fa50eb91f5da4edd7, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/86132cf5fbc949b1b31513a745152274] 2024-11-17T08:15:56,225 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting d37e6908cb3f4f0a96e3608dca56ed45, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731831352619 2024-11-17T08:15:56,226 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e0afce9248c4c679dabb0d639d5bd7c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731831352703 2024-11-17T08:15:56,226 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting ceeae6fb0a524d978b5e762a5ad33a4a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731831352845 2024-11-17T08:15:56,226 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a3c8d88746e4e5682ed23bede52e0b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731831354114 2024-11-17T08:15:56,227 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7545be14c1a432f83eeb367c7f7eb9c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731831352619 2024-11-17T08:15:56,231 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffb3418ecacf41ceae841c4e77d1e06f, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731831352703 2024-11-17T08:15:56,232 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ba8e8a75621419fa50eb91f5da4edd7, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731831352845 2024-11-17T08:15:56,232 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86132cf5fbc949b1b31513a745152274, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731831354114 2024-11-17T08:15:56,251 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b852432b87998249586edaadc388ac84#B#compaction#294 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:56,252 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/fb195d63e9754afaad855395aa5ad959 is 50, key is test_row_0/B:col10/1731831354793/Put/seqid=0 2024-11-17T08:15:56,257 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:15:56,277 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117f19faf145de940c597ece0fd989009d0_b852432b87998249586edaadc388ac84 store=[table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:15:56,280 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117f19faf145de940c597ece0fd989009d0_b852432b87998249586edaadc388ac84, store=[table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:15:56,281 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f19faf145de940c597ece0fd989009d0_b852432b87998249586edaadc388ac84 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:15:56,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742168_1344 (size=12139) 2024-11-17T08:15:56,307 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/fb195d63e9754afaad855395aa5ad959 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/fb195d63e9754afaad855395aa5ad959 2024-11-17T08:15:56,315 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b852432b87998249586edaadc388ac84/B of b852432b87998249586edaadc388ac84 into fb195d63e9754afaad855395aa5ad959(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:56,315 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b852432b87998249586edaadc388ac84: 2024-11-17T08:15:56,315 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., storeName=b852432b87998249586edaadc388ac84/B, priority=12, startTime=1731831356223; duration=0sec 2024-11-17T08:15:56,315 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:15:56,315 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b852432b87998249586edaadc388ac84:B 2024-11-17T08:15:56,315 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T08:15:56,316 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T08:15:56,316 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): b852432b87998249586edaadc388ac84/C is initiating minor compaction (all files) 2024-11-17T08:15:56,316 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b852432b87998249586edaadc388ac84/C in TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:56,317 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/9a33c619f6a44d20bdf8a75cff1338a5, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/d4078f8657454a7baf481a8fcf606cd9, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/2341bb1c27574b838b5c811ebca77caa, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/2d03be7081de4ee3a282d7070b4a3938] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp, totalSize=44.6 K 2024-11-17T08:15:56,317 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a33c619f6a44d20bdf8a75cff1338a5, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731831352619 2024-11-17T08:15:56,317 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting d4078f8657454a7baf481a8fcf606cd9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731831352703 2024-11-17T08:15:56,318 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 2341bb1c27574b838b5c811ebca77caa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731831352845 2024-11-17T08:15:56,318 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d03be7081de4ee3a282d7070b4a3938, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731831354114 2024-11-17T08:15:56,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on b852432b87998249586edaadc388ac84 2024-11-17T08:15:56,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b852432b87998249586edaadc388ac84 3/3 column families, dataSize=161.02 KB heapSize=447.23 KB 2024-11-17T08:15:56,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=A 2024-11-17T08:15:56,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:15:56,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=B 2024-11-17T08:15:56,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:15:56,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=C 2024-11-17T08:15:56,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:15:56,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742169_1345 (size=4469) 2024-11-17T08:15:56,335 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b852432b87998249586edaadc388ac84#A#compaction#295 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:56,335 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/f0ce9c39152a4968b0d38a4ccb8bc462 is 175, key is test_row_0/A:col10/1731831354793/Put/seqid=0 2024-11-17T08:15:56,342 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b852432b87998249586edaadc388ac84#C#compaction#296 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:56,342 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/5d81b47fdb964653a00df56c6a12395f is 50, key is test_row_0/C:col10/1731831354793/Put/seqid=0 2024-11-17T08:15:56,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411171bb0f4c8f35e49169e03f54343c71ef6_b852432b87998249586edaadc388ac84 is 50, key is test_row_0/A:col10/1731831355203/Put/seqid=0 2024-11-17T08:15:56,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:56,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831416374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:56,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:56,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:56,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:56,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:56,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53496 deadline: 1731831416374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:56,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:56,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:56,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:56,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:56,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831416379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:56,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:56,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:56,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:56,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742170_1346 (size=31093) 2024-11-17T08:15:56,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:56,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831416383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:56,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:56,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:56,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:56,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:56,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831416392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:56,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:56,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:56,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:56,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742172_1348 (size=14794) 2024-11-17T08:15:56,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742171_1347 (size=12139) 2024-11-17T08:15:56,444 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:56,456 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/5d81b47fdb964653a00df56c6a12395f as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/5d81b47fdb964653a00df56c6a12395f 2024-11-17T08:15:56,457 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411171bb0f4c8f35e49169e03f54343c71ef6_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411171bb0f4c8f35e49169e03f54343c71ef6_b852432b87998249586edaadc388ac84 2024-11-17T08:15:56,458 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/8f0b82ac584048f8a066bc031ec3dfab, store: [table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:15:56,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/8f0b82ac584048f8a066bc031ec3dfab is 175, key is test_row_0/A:col10/1731831355203/Put/seqid=0 2024-11-17T08:15:56,464 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b852432b87998249586edaadc388ac84/C of b852432b87998249586edaadc388ac84 into 5d81b47fdb964653a00df56c6a12395f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:56,464 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b852432b87998249586edaadc388ac84: 2024-11-17T08:15:56,464 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., storeName=b852432b87998249586edaadc388ac84/C, priority=12, startTime=1731831356223; duration=0sec 2024-11-17T08:15:56,464 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:56,464 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b852432b87998249586edaadc388ac84:C 2024-11-17T08:15:56,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742173_1349 (size=39749) 2024-11-17T08:15:56,502 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=204, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/8f0b82ac584048f8a066bc031ec3dfab 2024-11-17T08:15:56,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:56,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831416495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:56,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:56,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:56,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:56,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/6b058b0db96d4faa92357c9cc855cf69 is 50, key is test_row_0/B:col10/1731831355203/Put/seqid=0 2024-11-17T08:15:56,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:56,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831416513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:56,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:56,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:56,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:56,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742174_1350 (size=12151) 2024-11-17T08:15:56,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/6b058b0db96d4faa92357c9cc855cf69 2024-11-17T08:15:56,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/eadbac69351f48429df54b2d22b21fd9 is 50, key is test_row_0/C:col10/1731831355203/Put/seqid=0 2024-11-17T08:15:56,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742175_1351 (size=12151) 2024-11-17T08:15:56,574 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/eadbac69351f48429df54b2d22b21fd9 2024-11-17T08:15:56,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/8f0b82ac584048f8a066bc031ec3dfab as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/8f0b82ac584048f8a066bc031ec3dfab 2024-11-17T08:15:56,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/8f0b82ac584048f8a066bc031ec3dfab, entries=200, sequenceid=204, filesize=38.8 K 2024-11-17T08:15:56,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/6b058b0db96d4faa92357c9cc855cf69 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/6b058b0db96d4faa92357c9cc855cf69 2024-11-17T08:15:56,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/6b058b0db96d4faa92357c9cc855cf69, entries=150, sequenceid=204, filesize=11.9 K 2024-11-17T08:15:56,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/eadbac69351f48429df54b2d22b21fd9 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/eadbac69351f48429df54b2d22b21fd9 2024-11-17T08:15:56,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/eadbac69351f48429df54b2d22b21fd9, entries=150, sequenceid=204, filesize=11.9 K 2024-11-17T08:15:56,604 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-17T08:15:56,605 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~447.19 KB/457920, currentSize=26.84 KB/27480 for b852432b87998249586edaadc388ac84 in 278ms, sequenceid=204, compaction requested=false 2024-11-17T08:15:56,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b852432b87998249586edaadc388ac84: 2024-11-17T08:15:56,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on b852432b87998249586edaadc388ac84 2024-11-17T08:15:56,744 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b852432b87998249586edaadc388ac84 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T08:15:56,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=A 2024-11-17T08:15:56,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:56,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=B 2024-11-17T08:15:56,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:56,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=C 2024-11-17T08:15:56,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:56,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b73ceb5284c2479e97b1e503a546938d_b852432b87998249586edaadc388ac84 is 50, key is test_row_0/A:col10/1731831356743/Put/seqid=0 2024-11-17T08:15:56,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T08:15:56,769 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:15:56,771 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:15:56,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-17T08:15:56,774 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:15:56,775 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:15:56,775 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:15:56,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T08:15:56,801 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/f0ce9c39152a4968b0d38a4ccb8bc462 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/f0ce9c39152a4968b0d38a4ccb8bc462 2024-11-17T08:15:56,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742176_1352 (size=17284) 2024-11-17T08:15:56,808 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:56,812 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b852432b87998249586edaadc388ac84/A of b852432b87998249586edaadc388ac84 into f0ce9c39152a4968b0d38a4ccb8bc462(size=30.4 K), total size for store is 69.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:56,813 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b852432b87998249586edaadc388ac84: 2024-11-17T08:15:56,813 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., storeName=b852432b87998249586edaadc388ac84/A, priority=12, startTime=1731831356223; duration=0sec 2024-11-17T08:15:56,813 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:56,813 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b852432b87998249586edaadc388ac84:A 2024-11-17T08:15:56,818 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b73ceb5284c2479e97b1e503a546938d_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b73ceb5284c2479e97b1e503a546938d_b852432b87998249586edaadc388ac84 2024-11-17T08:15:56,819 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/586c4296929f42f38ce6178c20adb641, store: [table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:15:56,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/586c4296929f42f38ce6178c20adb641 is 175, key is test_row_0/A:col10/1731831356743/Put/seqid=0 2024-11-17T08:15:56,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742177_1353 (size=48389) 2024-11-17T08:15:56,837 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=215, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/586c4296929f42f38ce6178c20adb641 2024-11-17T08:15:56,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/b22077599bcd450d987ec1b24324f3e6 is 50, key is test_row_0/B:col10/1731831356743/Put/seqid=0 2024-11-17T08:15:56,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T08:15:56,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742178_1354 (size=12151) 2024-11-17T08:15:56,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/b22077599bcd450d987ec1b24324f3e6 2024-11-17T08:15:56,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/248732c3099d49aca3f167641c1779e7 is 50, key is test_row_0/C:col10/1731831356743/Put/seqid=0 2024-11-17T08:15:56,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731831356854, max timestamp=1731831356920Num uniques 859; 2024-11-17T08:15:56,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731831356854, max timestamp=1731831356920Num uniques 859; 2024-11-17T08:15:56,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T08:15:56,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T08:15:56,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731831356854, max timestamp=1731831356920Num uniques 859; 2024-11-17T08:15:56,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T08:15:56,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-17T08:15:56,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:56,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:15:56,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:56,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:56,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:56,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:56,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742179_1355 (size=12151) 2024-11-17T08:15:56,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/248732c3099d49aca3f167641c1779e7 2024-11-17T08:15:56,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/586c4296929f42f38ce6178c20adb641 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/586c4296929f42f38ce6178c20adb641 2024-11-17T08:15:56,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/586c4296929f42f38ce6178c20adb641, entries=250, sequenceid=215, filesize=47.3 K 2024-11-17T08:15:56,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/b22077599bcd450d987ec1b24324f3e6 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/b22077599bcd450d987ec1b24324f3e6 2024-11-17T08:15:56,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/b22077599bcd450d987ec1b24324f3e6, entries=150, sequenceid=215, filesize=11.9 K 2024-11-17T08:15:56,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/248732c3099d49aca3f167641c1779e7 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/248732c3099d49aca3f167641c1779e7 2024-11-17T08:15:56,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/248732c3099d49aca3f167641c1779e7, entries=150, sequenceid=215, filesize=11.9 K 2024-11-17T08:15:56,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=67.09 KB/68700 for b852432b87998249586edaadc388ac84 in 230ms, sequenceid=215, compaction requested=true 2024-11-17T08:15:56,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b852432b87998249586edaadc388ac84: 2024-11-17T08:15:56,975 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:15:56,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b852432b87998249586edaadc388ac84:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:15:56,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:56,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b852432b87998249586edaadc388ac84:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:15:56,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:56,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b852432b87998249586edaadc388ac84:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:15:56,975 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:15:56,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:15:56,977 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36441 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:15:56,977 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:15:56,977 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): b852432b87998249586edaadc388ac84/B is initiating minor compaction (all files) 2024-11-17T08:15:56,977 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): b852432b87998249586edaadc388ac84/A is initiating minor compaction (all files) 2024-11-17T08:15:56,978 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b852432b87998249586edaadc388ac84/B in TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:56,978 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b852432b87998249586edaadc388ac84/A in TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:56,978 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/fb195d63e9754afaad855395aa5ad959, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/6b058b0db96d4faa92357c9cc855cf69, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/b22077599bcd450d987ec1b24324f3e6] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp, totalSize=35.6 K 2024-11-17T08:15:56,978 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/f0ce9c39152a4968b0d38a4ccb8bc462, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/8f0b82ac584048f8a066bc031ec3dfab, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/586c4296929f42f38ce6178c20adb641] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp, totalSize=116.4 K 2024-11-17T08:15:56,978 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:56,978 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. files: [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/f0ce9c39152a4968b0d38a4ccb8bc462, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/8f0b82ac584048f8a066bc031ec3dfab, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/586c4296929f42f38ce6178c20adb641] 2024-11-17T08:15:56,978 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting fb195d63e9754afaad855395aa5ad959, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731831354114 2024-11-17T08:15:56,978 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0ce9c39152a4968b0d38a4ccb8bc462, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731831354114 2024-11-17T08:15:56,978 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b058b0db96d4faa92357c9cc855cf69, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731831355186 2024-11-17T08:15:56,978 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f0b82ac584048f8a066bc031ec3dfab, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731831355186 2024-11-17T08:15:56,978 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting b22077599bcd450d987ec1b24324f3e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731831356386 2024-11-17T08:15:56,979 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 586c4296929f42f38ce6178c20adb641, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731831356343 2024-11-17T08:15:56,979 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b852432b87998249586edaadc388ac84 3/3 column families, dataSize=73.80 KB heapSize=205.69 KB 2024-11-17T08:15:56,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on b852432b87998249586edaadc388ac84 2024-11-17T08:15:56,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=A 2024-11-17T08:15:56,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:15:56,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=B 2024-11-17T08:15:56,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:15:56,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=C 2024-11-17T08:15:56,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:15:57,001 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b852432b87998249586edaadc388ac84#B#compaction#303 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:57,002 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/12b9af7ad4a24faba40517eb010b06a8 is 50, key is test_row_0/B:col10/1731831356743/Put/seqid=0 2024-11-17T08:15:57,009 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:15:57,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117eb1e0c3031ca4e509d0be5d8700dbce5_b852432b87998249586edaadc388ac84 is 50, key is test_row_0/A:col10/1731831356977/Put/seqid=0 2024-11-17T08:15:57,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742180_1356 (size=12391) 2024-11-17T08:15:57,069 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411175a492a7996534403bdc5ddec520ead3f_b852432b87998249586edaadc388ac84 store=[table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:15:57,071 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411175a492a7996534403bdc5ddec520ead3f_b852432b87998249586edaadc388ac84, store=[table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:15:57,071 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411175a492a7996534403bdc5ddec520ead3f_b852432b87998249586edaadc388ac84 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:15:57,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-17T08:15:57,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:57,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:15:57,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:57,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:57,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:57,084 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/12b9af7ad4a24faba40517eb010b06a8 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/12b9af7ad4a24faba40517eb010b06a8 2024-11-17T08:15:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:57,089 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b852432b87998249586edaadc388ac84/B of b852432b87998249586edaadc388ac84 into 12b9af7ad4a24faba40517eb010b06a8(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:57,089 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b852432b87998249586edaadc388ac84: 2024-11-17T08:15:57,089 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., storeName=b852432b87998249586edaadc388ac84/B, priority=13, startTime=1731831356975; duration=0sec 2024-11-17T08:15:57,089 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:15:57,089 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b852432b87998249586edaadc388ac84:B 2024-11-17T08:15:57,089 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:15:57,090 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36441 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:15:57,090 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): b852432b87998249586edaadc388ac84/C is initiating minor compaction (all files) 2024-11-17T08:15:57,090 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b852432b87998249586edaadc388ac84/C in TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:57,090 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/5d81b47fdb964653a00df56c6a12395f, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/eadbac69351f48429df54b2d22b21fd9, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/248732c3099d49aca3f167641c1779e7] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp, totalSize=35.6 K 2024-11-17T08:15:57,090 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d81b47fdb964653a00df56c6a12395f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731831354114 2024-11-17T08:15:57,091 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting eadbac69351f48429df54b2d22b21fd9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731831355186 2024-11-17T08:15:57,091 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 248732c3099d49aca3f167641c1779e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731831356386 2024-11-17T08:15:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T08:15:57,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742181_1357 (size=14794) 2024-11-17T08:15:57,135 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b852432b87998249586edaadc388ac84#C#compaction#306 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:57,136 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/6e51d6dbe51c4b55bdb7136e3bc7ed73 is 50, key is test_row_0/C:col10/1731831356743/Put/seqid=0 2024-11-17T08:15:57,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742182_1358 (size=4469) 2024-11-17T08:15:57,141 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b852432b87998249586edaadc388ac84#A#compaction#304 average throughput is 0.19 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:15:57,142 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/bbbc3416a9f94ef788e9beff130ff1ea is 175, key is test_row_0/A:col10/1731831356743/Put/seqid=0 2024-11-17T08:15:57,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742184_1360 (size=31345) 2024-11-17T08:15:57,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742183_1359 (size=12391) 2024-11-17T08:15:57,222 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/bbbc3416a9f94ef788e9beff130ff1ea as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/bbbc3416a9f94ef788e9beff130ff1ea 2024-11-17T08:15:57,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:57,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831417196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:57,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:57,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:57,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:57,233 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/6e51d6dbe51c4b55bdb7136e3bc7ed73 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/6e51d6dbe51c4b55bdb7136e3bc7ed73 2024-11-17T08:15:57,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-17T08:15:57,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:57,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:15:57,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:57,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:57,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:57,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:57,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831417224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:57,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:57,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:57,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:57,274 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b852432b87998249586edaadc388ac84/A of b852432b87998249586edaadc388ac84 into bbbc3416a9f94ef788e9beff130ff1ea(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:57,274 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b852432b87998249586edaadc388ac84: 2024-11-17T08:15:57,274 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., storeName=b852432b87998249586edaadc388ac84/A, priority=13, startTime=1731831356975; duration=0sec 2024-11-17T08:15:57,274 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:57,275 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b852432b87998249586edaadc388ac84:A 2024-11-17T08:15:57,283 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b852432b87998249586edaadc388ac84/C of b852432b87998249586edaadc388ac84 into 6e51d6dbe51c4b55bdb7136e3bc7ed73(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:15:57,283 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b852432b87998249586edaadc388ac84: 2024-11-17T08:15:57,283 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., storeName=b852432b87998249586edaadc388ac84/C, priority=13, startTime=1731831356975; duration=0sec 2024-11-17T08:15:57,283 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:15:57,283 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b852432b87998249586edaadc388ac84:C 2024-11-17T08:15:57,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831417335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:57,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:57,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:57,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:57,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:57,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831417355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:57,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:57,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:57,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:57,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:57,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53496 deadline: 1731831417394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:57,402 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e3d0adb87c01,40631,1731831247805 for 'test_row_0' in b852432b87998249586edaadc388ac84 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T08:15:57,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:57,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:57,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:57,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T08:15:57,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831417403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:57,415 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to e3d0adb87c01,40631,1731831247805 for 'test_row_1' in b852432b87998249586edaadc388ac84 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T08:15:57,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:57,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:57,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:57,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831417414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:57,420 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to e3d0adb87c01,40631,1731831247805 for 'test_row_0' in b852432b87998249586edaadc388ac84 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T08:15:57,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:57,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:57,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:57,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-17T08:15:57,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:57,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:15:57,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:57,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:57,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:57,519 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:57,533 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117eb1e0c3031ca4e509d0be5d8700dbce5_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117eb1e0c3031ca4e509d0be5d8700dbce5_b852432b87998249586edaadc388ac84 2024-11-17T08:15:57,534 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/0766e4b1e3954c048cf194768da5d135, store: [table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:15:57,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/0766e4b1e3954c048cf194768da5d135 is 175, key is test_row_0/A:col10/1731831356977/Put/seqid=0 2024-11-17T08:15:57,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:57,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831417554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:57,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:57,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:57,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:57,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-17T08:15:57,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:57,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:15:57,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:57,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:57,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:57,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:57,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831417573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:57,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:57,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:57,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:57,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:57,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742185_1361 (size=39749) 2024-11-17T08:15:57,604 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=244, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/0766e4b1e3954c048cf194768da5d135 2024-11-17T08:15:57,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/a6381decb42341bea2ce07f69368babf is 50, key is test_row_0/B:col10/1731831356977/Put/seqid=0 2024-11-17T08:15:57,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742186_1362 (size=12151) 2024-11-17T08:15:57,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/a6381decb42341bea2ce07f69368babf 2024-11-17T08:15:57,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/a8ba04f8e6fd45b19cae086707ec3c1d is 50, key is test_row_0/C:col10/1731831356977/Put/seqid=0 2024-11-17T08:15:57,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742187_1363 (size=12151) 2024-11-17T08:15:57,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-17T08:15:57,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:57,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:15:57,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:57,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:57,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:57,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:57,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:57,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831417874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:57,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:57,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:57,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:57,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-17T08:15:57,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:57,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:15:57,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:57,891 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:57,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:57,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:57,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:57,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831417894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:57,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:57,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:57,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:57,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T08:15:58,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-17T08:15:58,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:58,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:15:58,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:58,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:58,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:58,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:15:58,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/a8ba04f8e6fd45b19cae086707ec3c1d 2024-11-17T08:15:58,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/0766e4b1e3954c048cf194768da5d135 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/0766e4b1e3954c048cf194768da5d135 2024-11-17T08:15:58,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/0766e4b1e3954c048cf194768da5d135, entries=200, sequenceid=244, filesize=38.8 K 2024-11-17T08:15:58,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/a6381decb42341bea2ce07f69368babf as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/a6381decb42341bea2ce07f69368babf 2024-11-17T08:15:58,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/a6381decb42341bea2ce07f69368babf, entries=150, sequenceid=244, filesize=11.9 K 2024-11-17T08:15:58,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/a8ba04f8e6fd45b19cae086707ec3c1d as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/a8ba04f8e6fd45b19cae086707ec3c1d 2024-11-17T08:15:58,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/a8ba04f8e6fd45b19cae086707ec3c1d, entries=150, sequenceid=244, filesize=11.9 K 2024-11-17T08:15:58,093 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~205.64 KB/210576, currentSize=107.34 KB/109920 for b852432b87998249586edaadc388ac84 in 1113ms, sequenceid=244, compaction requested=false 2024-11-17T08:15:58,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b852432b87998249586edaadc388ac84: 2024-11-17T08:15:58,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-17T08:15:58,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:58,199 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing b852432b87998249586edaadc388ac84 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-17T08:15:58,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=A 2024-11-17T08:15:58,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:58,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=B 2024-11-17T08:15:58,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:58,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=C 2024-11-17T08:15:58,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:58,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111789df79b307124da18285257665d4ab0c_b852432b87998249586edaadc388ac84 is 50, key is test_row_0/A:col10/1731831357153/Put/seqid=0 2024-11-17T08:15:58,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742188_1364 (size=12454) 2024-11-17T08:15:58,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:58,279 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111789df79b307124da18285257665d4ab0c_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111789df79b307124da18285257665d4ab0c_b852432b87998249586edaadc388ac84 2024-11-17T08:15:58,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/ba926f3391484103bdaa5af0bda39bc3, store: [table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:15:58,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/ba926f3391484103bdaa5af0bda39bc3 is 175, key is test_row_0/A:col10/1731831357153/Put/seqid=0 2024-11-17T08:15:58,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742189_1365 (size=31255) 2024-11-17T08:15:58,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on b852432b87998249586edaadc388ac84 2024-11-17T08:15:58,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:15:58,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:58,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831418444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:58,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:58,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:58,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:58,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:58,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831418445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:58,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:58,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:58,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:58,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:58,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831418564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:58,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:58,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:58,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831418564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:58,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:58,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:58,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:58,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:58,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:58,733 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=266, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/ba926f3391484103bdaa5af0bda39bc3 2024-11-17T08:15:58,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/4735c6dc7f5a47ecbd9397f14a519513 is 50, key is test_row_0/B:col10/1731831357153/Put/seqid=0 2024-11-17T08:15:58,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:58,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831418784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:58,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:58,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831418784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:58,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:58,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:58,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:58,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:58,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:58,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:58,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742190_1366 (size=12301) 2024-11-17T08:15:58,802 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/4735c6dc7f5a47ecbd9397f14a519513 2024-11-17T08:15:58,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/e1882648e79e4860b40eaac828d576f8 is 50, key is test_row_0/C:col10/1731831357153/Put/seqid=0 2024-11-17T08:15:58,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742191_1367 (size=12301) 2024-11-17T08:15:58,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T08:15:59,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:59,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831419104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:59,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:59,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831419104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:59,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:59,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:59,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:59,272 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/e1882648e79e4860b40eaac828d576f8 2024-11-17T08:15:59,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/ba926f3391484103bdaa5af0bda39bc3 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/ba926f3391484103bdaa5af0bda39bc3 2024-11-17T08:15:59,284 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/ba926f3391484103bdaa5af0bda39bc3, entries=150, sequenceid=266, filesize=30.5 K 2024-11-17T08:15:59,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/4735c6dc7f5a47ecbd9397f14a519513 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/4735c6dc7f5a47ecbd9397f14a519513 2024-11-17T08:15:59,290 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/4735c6dc7f5a47ecbd9397f14a519513, entries=150, sequenceid=266, filesize=12.0 K 2024-11-17T08:15:59,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/e1882648e79e4860b40eaac828d576f8 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/e1882648e79e4860b40eaac828d576f8 2024-11-17T08:15:59,298 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/e1882648e79e4860b40eaac828d576f8, entries=150, sequenceid=266, filesize=12.0 K 2024-11-17T08:15:59,300 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for b852432b87998249586edaadc388ac84 in 1100ms, sequenceid=266, compaction requested=true 2024-11-17T08:15:59,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for b852432b87998249586edaadc388ac84: 2024-11-17T08:15:59,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:15:59,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-17T08:15:59,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-17T08:15:59,307 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-17T08:15:59,307 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5270 sec 2024-11-17T08:15:59,309 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.5370 sec 2024-11-17T08:15:59,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on b852432b87998249586edaadc388ac84 2024-11-17T08:15:59,430 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b852432b87998249586edaadc388ac84 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-17T08:15:59,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=A 2024-11-17T08:15:59,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:59,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=B 2024-11-17T08:15:59,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:59,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=C 2024-11-17T08:15:59,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:15:59,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d89eabba5dea4b5d82f1c3a8dbd9ea5c_b852432b87998249586edaadc388ac84 is 50, key is test_row_0/A:col10/1731831359429/Put/seqid=0 2024-11-17T08:15:59,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742192_1368 (size=14994) 2024-11-17T08:15:59,448 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:15:59,458 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d89eabba5dea4b5d82f1c3a8dbd9ea5c_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d89eabba5dea4b5d82f1c3a8dbd9ea5c_b852432b87998249586edaadc388ac84 2024-11-17T08:15:59,459 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/d1409acfca2741a7b2cd1c87f8e75fe1, store: [table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:15:59,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/d1409acfca2741a7b2cd1c87f8e75fe1 is 175, key is test_row_0/A:col10/1731831359429/Put/seqid=0 2024-11-17T08:15:59,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742193_1369 (size=39949) 2024-11-17T08:15:59,465 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=280, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/d1409acfca2741a7b2cd1c87f8e75fe1 2024-11-17T08:15:59,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/d8ddea08cc604e0fb91311f82aad4ceb is 50, key is test_row_0/B:col10/1731831359429/Put/seqid=0 2024-11-17T08:15:59,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742194_1370 (size=12301) 2024-11-17T08:15:59,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:59,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831419497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:59,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:59,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:59,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831419497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:59,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:59,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:59,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:59,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:59,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53496 deadline: 1731831419503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:59,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:59,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:59,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:59,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:59,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:59,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:59,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831419623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:59,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:59,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:59,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:59,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831419623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:59,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:59,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:59,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:59,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:59,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:59,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831419623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:59,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:59,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:59,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:59,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:59,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831419623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:59,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:59,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:59,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53496 deadline: 1731831419623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:59,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:59,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:59,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:59,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:59,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:59,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:59,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831419834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:59,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:59,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:59,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:59,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:59,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831419834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:59,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:59,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:59,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:59,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:15:59,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53496 deadline: 1731831419835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:15:59,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:59,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:15:59,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:15:59,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/d8ddea08cc604e0fb91311f82aad4ceb 2024-11-17T08:15:59,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/20ad3be8a0e14e7cb3639a9fdde3c762 is 50, key is test_row_0/C:col10/1731831359429/Put/seqid=0 2024-11-17T08:15:59,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742195_1371 (size=12301) 2024-11-17T08:16:00,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:00,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831420143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:00,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:00,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831420144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:00,149 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:00,149 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:00,149 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:00,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:00,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:00,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:00,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53496 deadline: 1731831420144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:00,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:00,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:00,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:00,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:00,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/20ad3be8a0e14e7cb3639a9fdde3c762 2024-11-17T08:16:00,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/d1409acfca2741a7b2cd1c87f8e75fe1 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/d1409acfca2741a7b2cd1c87f8e75fe1 2024-11-17T08:16:00,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/d1409acfca2741a7b2cd1c87f8e75fe1, entries=200, sequenceid=280, filesize=39.0 K 2024-11-17T08:16:00,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/d8ddea08cc604e0fb91311f82aad4ceb as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/d8ddea08cc604e0fb91311f82aad4ceb 2024-11-17T08:16:00,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/d8ddea08cc604e0fb91311f82aad4ceb, entries=150, sequenceid=280, filesize=12.0 K 2024-11-17T08:16:00,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/20ad3be8a0e14e7cb3639a9fdde3c762 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/20ad3be8a0e14e7cb3639a9fdde3c762 2024-11-17T08:16:00,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/20ad3be8a0e14e7cb3639a9fdde3c762, entries=150, sequenceid=280, filesize=12.0 K 2024-11-17T08:16:00,349 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for b852432b87998249586edaadc388ac84 in 918ms, sequenceid=280, compaction requested=true 2024-11-17T08:16:00,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b852432b87998249586edaadc388ac84: 2024-11-17T08:16:00,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b852432b87998249586edaadc388ac84:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:16:00,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:00,349 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T08:16:00,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b852432b87998249586edaadc388ac84:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:16:00,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:00,349 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T08:16:00,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b852432b87998249586edaadc388ac84:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:16:00,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:16:00,350 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142298 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T08:16:00,350 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49144 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T08:16:00,350 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): b852432b87998249586edaadc388ac84/B is initiating minor compaction (all files) 2024-11-17T08:16:00,350 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): b852432b87998249586edaadc388ac84/A is initiating minor compaction (all files) 2024-11-17T08:16:00,350 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b852432b87998249586edaadc388ac84/A in TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:00,350 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b852432b87998249586edaadc388ac84/B in TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:00,350 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/bbbc3416a9f94ef788e9beff130ff1ea, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/0766e4b1e3954c048cf194768da5d135, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/ba926f3391484103bdaa5af0bda39bc3, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/d1409acfca2741a7b2cd1c87f8e75fe1] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp, totalSize=139.0 K 2024-11-17T08:16:00,350 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/12b9af7ad4a24faba40517eb010b06a8, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/a6381decb42341bea2ce07f69368babf, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/4735c6dc7f5a47ecbd9397f14a519513, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/d8ddea08cc604e0fb91311f82aad4ceb] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp, totalSize=48.0 K 2024-11-17T08:16:00,350 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:00,350 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. files: [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/bbbc3416a9f94ef788e9beff130ff1ea, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/0766e4b1e3954c048cf194768da5d135, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/ba926f3391484103bdaa5af0bda39bc3, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/d1409acfca2741a7b2cd1c87f8e75fe1] 2024-11-17T08:16:00,350 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 12b9af7ad4a24faba40517eb010b06a8, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731831356386 2024-11-17T08:16:00,350 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbbc3416a9f94ef788e9beff130ff1ea, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731831356386 2024-11-17T08:16:00,350 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting a6381decb42341bea2ce07f69368babf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731831356948 2024-11-17T08:16:00,350 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0766e4b1e3954c048cf194768da5d135, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731831356947 2024-11-17T08:16:00,351 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 4735c6dc7f5a47ecbd9397f14a519513, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731831357153 2024-11-17T08:16:00,351 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba926f3391484103bdaa5af0bda39bc3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731831357153 2024-11-17T08:16:00,351 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting d8ddea08cc604e0fb91311f82aad4ceb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731831358442 2024-11-17T08:16:00,351 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1409acfca2741a7b2cd1c87f8e75fe1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731831358437 2024-11-17T08:16:00,356 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:16:00,361 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411179c28cf81967c4e67b6d5d534c91fa858_b852432b87998249586edaadc388ac84 store=[table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:16:00,361 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b852432b87998249586edaadc388ac84#B#compaction#316 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:00,362 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/9cf4d97e5beb49629fd5bdacb34ce11a is 50, key is test_row_0/B:col10/1731831359429/Put/seqid=0 2024-11-17T08:16:00,364 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411179c28cf81967c4e67b6d5d534c91fa858_b852432b87998249586edaadc388ac84, store=[table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:16:00,364 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411179c28cf81967c4e67b6d5d534c91fa858_b852432b87998249586edaadc388ac84 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:16:00,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742196_1372 (size=12677) 2024-11-17T08:16:00,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742197_1373 (size=4469) 2024-11-17T08:16:00,374 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/9cf4d97e5beb49629fd5bdacb34ce11a as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/9cf4d97e5beb49629fd5bdacb34ce11a 2024-11-17T08:16:00,377 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b852432b87998249586edaadc388ac84/B of b852432b87998249586edaadc388ac84 into 9cf4d97e5beb49629fd5bdacb34ce11a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:00,377 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b852432b87998249586edaadc388ac84: 2024-11-17T08:16:00,377 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., storeName=b852432b87998249586edaadc388ac84/B, priority=12, startTime=1731831360349; duration=0sec 2024-11-17T08:16:00,378 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:16:00,378 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b852432b87998249586edaadc388ac84:B 2024-11-17T08:16:00,378 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T08:16:00,380 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49144 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T08:16:00,380 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): b852432b87998249586edaadc388ac84/C is initiating minor compaction (all files) 2024-11-17T08:16:00,380 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b852432b87998249586edaadc388ac84/C in TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:00,380 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/6e51d6dbe51c4b55bdb7136e3bc7ed73, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/a8ba04f8e6fd45b19cae086707ec3c1d, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/e1882648e79e4860b40eaac828d576f8, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/20ad3be8a0e14e7cb3639a9fdde3c762] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp, totalSize=48.0 K 2024-11-17T08:16:00,381 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e51d6dbe51c4b55bdb7136e3bc7ed73, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731831356386 2024-11-17T08:16:00,381 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting a8ba04f8e6fd45b19cae086707ec3c1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731831356948 2024-11-17T08:16:00,381 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting e1882648e79e4860b40eaac828d576f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731831357153 2024-11-17T08:16:00,381 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 20ad3be8a0e14e7cb3639a9fdde3c762, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731831358442 2024-11-17T08:16:00,389 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b852432b87998249586edaadc388ac84#C#compaction#317 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:00,389 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/ddbd2a788abd49bfabf7beb82de1793e is 50, key is test_row_0/C:col10/1731831359429/Put/seqid=0 2024-11-17T08:16:00,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742198_1374 (size=12677) 2024-11-17T08:16:00,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on b852432b87998249586edaadc388ac84 2024-11-17T08:16:00,640 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b852432b87998249586edaadc388ac84 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-17T08:16:00,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=A 2024-11-17T08:16:00,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:00,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=B 2024-11-17T08:16:00,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:00,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=C 2024-11-17T08:16:00,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:00,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411178efd394900de40809710f23f8764c1c0_b852432b87998249586edaadc388ac84 is 50, key is test_row_0/A:col10/1731831359502/Put/seqid=0 2024-11-17T08:16:00,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742199_1375 (size=14994) 2024-11-17T08:16:00,662 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:00,665 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411178efd394900de40809710f23f8764c1c0_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178efd394900de40809710f23f8764c1c0_b852432b87998249586edaadc388ac84 2024-11-17T08:16:00,666 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/9b201c52a1b3495e9e6855cdf2d626eb, store: [table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:16:00,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/9b201c52a1b3495e9e6855cdf2d626eb is 175, key is test_row_0/A:col10/1731831359502/Put/seqid=0 2024-11-17T08:16:00,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742200_1376 (size=39949) 2024-11-17T08:16:00,677 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=300, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/9b201c52a1b3495e9e6855cdf2d626eb 2024-11-17T08:16:00,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:00,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831420667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:00,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:00,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53496 deadline: 1731831420667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:00,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:00,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:00,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:00,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:00,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:00,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831420668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:00,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:00,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:00,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:00,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:00,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831420673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:00,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/259cc73715da44279dc50522a99ce362 is 50, key is test_row_0/B:col10/1731831359502/Put/seqid=0 2024-11-17T08:16:00,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:00,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:00,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:00,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742201_1377 (size=12301) 2024-11-17T08:16:00,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:00,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831420675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:00,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:00,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:00,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:00,771 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b852432b87998249586edaadc388ac84#A#compaction#315 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:00,772 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/10778409e00a4799a2c99c00a0eb3b8b is 175, key is test_row_0/A:col10/1731831359429/Put/seqid=0 2024-11-17T08:16:00,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742202_1378 (size=31631) 2024-11-17T08:16:00,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:00,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831420783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:00,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:00,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:00,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:00,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53496 deadline: 1731831420784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:00,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:00,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:00,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:00,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:00,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831420784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:00,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:00,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:00,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:00,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:00,797 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/ddbd2a788abd49bfabf7beb82de1793e as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/ddbd2a788abd49bfabf7beb82de1793e 2024-11-17T08:16:00,801 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b852432b87998249586edaadc388ac84/C of b852432b87998249586edaadc388ac84 into ddbd2a788abd49bfabf7beb82de1793e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:00,801 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b852432b87998249586edaadc388ac84: 2024-11-17T08:16:00,801 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., storeName=b852432b87998249586edaadc388ac84/C, priority=12, startTime=1731831360349; duration=0sec 2024-11-17T08:16:00,801 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:00,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:00,801 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b852432b87998249586edaadc388ac84:C 2024-11-17T08:16:00,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831420793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:00,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:00,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:00,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:00,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:00,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831420803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:00,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:00,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:00,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:00,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T08:16:00,944 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:16:00,944 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:16:00,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-17T08:16:00,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-17T08:16:00,946 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:16:00,947 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:16:00,947 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:16:01,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:01,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831420996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:01,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:01,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53496 deadline: 1731831420996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:01,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:01,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:01,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:01,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:01,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:01,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:01,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:01,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831421004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:01,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:01,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:01,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:01,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:01,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831421004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:01,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:01,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:01,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:01,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:01,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831421023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:01,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:01,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:01,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:01,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-17T08:16:01,089 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/259cc73715da44279dc50522a99ce362 2024-11-17T08:16:01,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-17T08:16:01,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:01,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:16:01,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:01,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:01,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:01,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:01,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/289da5810f6346a3939eead8866dde14 is 50, key is test_row_0/C:col10/1731831359502/Put/seqid=0 2024-11-17T08:16:01,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742203_1379 (size=12301) 2024-11-17T08:16:01,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/289da5810f6346a3939eead8866dde14 2024-11-17T08:16:01,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/9b201c52a1b3495e9e6855cdf2d626eb as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/9b201c52a1b3495e9e6855cdf2d626eb 2024-11-17T08:16:01,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/9b201c52a1b3495e9e6855cdf2d626eb, entries=200, sequenceid=300, filesize=39.0 K 2024-11-17T08:16:01,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/259cc73715da44279dc50522a99ce362 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/259cc73715da44279dc50522a99ce362 2024-11-17T08:16:01,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/259cc73715da44279dc50522a99ce362, entries=150, sequenceid=300, filesize=12.0 K 2024-11-17T08:16:01,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/289da5810f6346a3939eead8866dde14 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/289da5810f6346a3939eead8866dde14 2024-11-17T08:16:01,181 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/10778409e00a4799a2c99c00a0eb3b8b as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/10778409e00a4799a2c99c00a0eb3b8b 2024-11-17T08:16:01,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/289da5810f6346a3939eead8866dde14, entries=150, sequenceid=300, filesize=12.0 K 2024-11-17T08:16:01,184 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for b852432b87998249586edaadc388ac84 in 544ms, sequenceid=300, compaction requested=false 2024-11-17T08:16:01,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b852432b87998249586edaadc388ac84: 2024-11-17T08:16:01,188 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b852432b87998249586edaadc388ac84/A of b852432b87998249586edaadc388ac84 into 10778409e00a4799a2c99c00a0eb3b8b(size=30.9 K), total size for store is 69.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:01,188 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b852432b87998249586edaadc388ac84: 2024-11-17T08:16:01,188 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., storeName=b852432b87998249586edaadc388ac84/A, priority=12, startTime=1731831360349; duration=0sec 2024-11-17T08:16:01,188 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:01,188 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b852432b87998249586edaadc388ac84:A 2024-11-17T08:16:01,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-17T08:16:01,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:01,257 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing b852432b87998249586edaadc388ac84 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-17T08:16:01,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=A 2024-11-17T08:16:01,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:01,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=B 2024-11-17T08:16:01,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:01,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=C 2024-11-17T08:16:01,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:01,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-17T08:16:01,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b7b03d1bf7b14b098c8a4066a6e6fd44_b852432b87998249586edaadc388ac84 is 50, key is test_row_0/A:col10/1731831360673/Put/seqid=0 2024-11-17T08:16:01,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on b852432b87998249586edaadc388ac84 2024-11-17T08:16:01,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:16:01,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742204_1380 (size=12454) 2024-11-17T08:16:01,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:01,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831421364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:01,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:01,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:01,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:01,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:01,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831421368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:01,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:01,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:01,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:01,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:01,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831421376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:01,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:01,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:01,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:01,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:01,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831421388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:01,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:01,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:01,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:01,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:01,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53496 deadline: 1731831421388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:01,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:01,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:01,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:01,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:01,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831421493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:01,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:01,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:01,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:01,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:01,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831421493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:01,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:01,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831421495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:01,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:01,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:01,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:01,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:01,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:01,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:01,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:01,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831421514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:01,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:01,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:01,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:01,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:01,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53496 deadline: 1731831421516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:01,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:01,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:01,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:01,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-17T08:16:01,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:01,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831421703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:01,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:01,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:01,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:01,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831421704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:01,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:01,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:01,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:01,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:01,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:01,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831421714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:01,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:01,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:01,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:01,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:01,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831421723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:01,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:01,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:01,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:01,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53496 deadline: 1731831421724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:01,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:01,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:01,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:01,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:01,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:01,743 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b7b03d1bf7b14b098c8a4066a6e6fd44_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b7b03d1bf7b14b098c8a4066a6e6fd44_b852432b87998249586edaadc388ac84 2024-11-17T08:16:01,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/f5519ad431ce49d9843ef756ee056435, store: [table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:16:01,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/f5519ad431ce49d9843ef756ee056435 is 175, key is test_row_0/A:col10/1731831360673/Put/seqid=0 2024-11-17T08:16:01,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742205_1381 (size=31255) 2024-11-17T08:16:02,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:02,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831422023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:02,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:02,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831422024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:02,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:02,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:02,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:02,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:02,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:02,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:02,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831422024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:02,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:02,033 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:02,033 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:02,033 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:02,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:02,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831422043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:02,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:02,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:02,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:02,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53496 deadline: 1731831422043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:02,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:02,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:02,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:02,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:02,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-17T08:16:02,147 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=315, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/f5519ad431ce49d9843ef756ee056435 2024-11-17T08:16:02,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/e8e89fd14b534b168ed4dfb3f0908d4b is 50, key is test_row_0/B:col10/1731831360673/Put/seqid=0 2024-11-17T08:16:02,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742206_1382 (size=12301) 2024-11-17T08:16:02,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:02,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831422544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:02,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:02,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831422544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:02,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:02,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:02,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:02,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:02,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:02,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831422544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:02,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:02,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:02,548 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:02,548 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:02,548 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:02,557 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/e8e89fd14b534b168ed4dfb3f0908d4b 2024-11-17T08:16:02,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:02,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831422553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:02,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:02,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:02,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:02,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/820d985e048d4130b3f3507147c1861d is 50, key is test_row_0/C:col10/1731831360673/Put/seqid=0 2024-11-17T08:16:02,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742207_1383 (size=12301) 2024-11-17T08:16:02,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:02,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53496 deadline: 1731831422564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:02,965 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/820d985e048d4130b3f3507147c1861d 2024-11-17T08:16:02,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/f5519ad431ce49d9843ef756ee056435 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/f5519ad431ce49d9843ef756ee056435 2024-11-17T08:16:02,972 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/f5519ad431ce49d9843ef756ee056435, entries=150, sequenceid=315, filesize=30.5 K 2024-11-17T08:16:02,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/e8e89fd14b534b168ed4dfb3f0908d4b as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/e8e89fd14b534b168ed4dfb3f0908d4b 2024-11-17T08:16:02,975 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/e8e89fd14b534b168ed4dfb3f0908d4b, entries=150, sequenceid=315, filesize=12.0 K 2024-11-17T08:16:02,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/820d985e048d4130b3f3507147c1861d as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/820d985e048d4130b3f3507147c1861d 2024-11-17T08:16:02,980 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/820d985e048d4130b3f3507147c1861d, entries=150, sequenceid=315, filesize=12.0 K 2024-11-17T08:16:02,981 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for b852432b87998249586edaadc388ac84 in 1723ms, sequenceid=315, compaction requested=true 2024-11-17T08:16:02,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for b852432b87998249586edaadc388ac84: 2024-11-17T08:16:02,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:02,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-17T08:16:02,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-17T08:16:02,984 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-17T08:16:02,984 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0350 sec 2024-11-17T08:16:02,985 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.0400 sec 2024-11-17T08:16:03,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-17T08:16:03,094 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:16:03,094 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:16:03,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-17T08:16:03,096 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:16:03,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T08:16:03,096 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:16:03,096 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:16:03,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T08:16:03,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-17T08:16:03,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:03,248 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing b852432b87998249586edaadc388ac84 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-17T08:16:03,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=A 2024-11-17T08:16:03,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:03,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=B 2024-11-17T08:16:03,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:03,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=C 2024-11-17T08:16:03,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:03,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411176f0361e12a9b409e914d6e9450fc28fb_b852432b87998249586edaadc388ac84 is 50, key is test_row_0/A:col10/1731831361384/Put/seqid=0 2024-11-17T08:16:03,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742208_1384 (size=12454) 2024-11-17T08:16:03,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:03,266 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411176f0361e12a9b409e914d6e9450fc28fb_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176f0361e12a9b409e914d6e9450fc28fb_b852432b87998249586edaadc388ac84 2024-11-17T08:16:03,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/733c378e5c0946bd91d2c23618ff5e98, store: [table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:16:03,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/733c378e5c0946bd91d2c23618ff5e98 is 175, key is test_row_0/A:col10/1731831361384/Put/seqid=0 2024-11-17T08:16:03,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742209_1385 (size=31255) 2024-11-17T08:16:03,275 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=335, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/733c378e5c0946bd91d2c23618ff5e98 2024-11-17T08:16:03,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/f3550537109449469a45ec194c42e48e is 50, key is test_row_0/B:col10/1731831361384/Put/seqid=0 2024-11-17T08:16:03,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742210_1386 (size=12301) 2024-11-17T08:16:03,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T08:16:03,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on b852432b87998249586edaadc388ac84 2024-11-17T08:16:03,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:16:03,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:03,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831423577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:03,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:03,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:03,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53496 deadline: 1731831423583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:03,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:03,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:03,596 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e3d0adb87c01,40631,1731831247805 for 'test_row_1' in b852432b87998249586edaadc388ac84 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T08:16:03,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:03,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:03,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:03,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:03,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831423593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:03,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:03,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:03,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:03,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:03,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831423595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:03,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:03,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:03,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:03,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:03,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831423596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:03,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:03,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:03,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:03,687 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/f3550537109449469a45ec194c42e48e 2024-11-17T08:16:03,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/db11d2e71b6e476687e6d0d2aaae25c5 is 50, key is test_row_0/C:col10/1731831361384/Put/seqid=0 2024-11-17T08:16:03,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742211_1387 (size=12301) 2024-11-17T08:16:03,701 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/db11d2e71b6e476687e6d0d2aaae25c5 2024-11-17T08:16:03,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/733c378e5c0946bd91d2c23618ff5e98 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/733c378e5c0946bd91d2c23618ff5e98 2024-11-17T08:16:03,712 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/733c378e5c0946bd91d2c23618ff5e98, entries=150, sequenceid=335, filesize=30.5 K 2024-11-17T08:16:03,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/f3550537109449469a45ec194c42e48e as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/f3550537109449469a45ec194c42e48e 2024-11-17T08:16:03,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:03,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831423704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:03,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:03,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831423704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:03,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:03,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:03,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:03,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:03,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:03,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:03,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:03,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831423715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:03,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:03,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831423715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:03,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:03,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:03,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:03,718 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/f3550537109449469a45ec194c42e48e, entries=150, sequenceid=335, filesize=12.0 K 2024-11-17T08:16:03,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:03,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:03,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:03,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/db11d2e71b6e476687e6d0d2aaae25c5 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/db11d2e71b6e476687e6d0d2aaae25c5 2024-11-17T08:16:03,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T08:16:03,725 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/db11d2e71b6e476687e6d0d2aaae25c5, entries=150, sequenceid=335, filesize=12.0 K 2024-11-17T08:16:03,728 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for b852432b87998249586edaadc388ac84 in 480ms, sequenceid=335, compaction requested=true 2024-11-17T08:16:03,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for b852432b87998249586edaadc388ac84: 2024-11-17T08:16:03,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:03,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-17T08:16:03,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-17T08:16:03,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-17T08:16:03,732 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 634 msec 2024-11-17T08:16:03,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 638 msec 2024-11-17T08:16:03,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on b852432b87998249586edaadc388ac84 2024-11-17T08:16:03,934 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b852432b87998249586edaadc388ac84 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-17T08:16:03,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=A 2024-11-17T08:16:03,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:03,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=B 2024-11-17T08:16:03,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:03,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=C 2024-11-17T08:16:03,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:03,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411176ec529d7aeab46db9846b5f729e79f4e_b852432b87998249586edaadc388ac84 is 50, key is test_row_0/A:col10/1731831363594/Put/seqid=0 2024-11-17T08:16:03,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742212_1388 (size=14994) 2024-11-17T08:16:03,956 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:03,960 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411176ec529d7aeab46db9846b5f729e79f4e_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176ec529d7aeab46db9846b5f729e79f4e_b852432b87998249586edaadc388ac84 2024-11-17T08:16:03,961 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/549a17f8fb734590bd54ae7cbc63f3ab, store: [table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:16:03,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/549a17f8fb734590bd54ae7cbc63f3ab is 175, key is test_row_0/A:col10/1731831363594/Put/seqid=0 2024-11-17T08:16:04,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742213_1389 (size=39949) 2024-11-17T08:16:04,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:04,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831424010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:04,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:04,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831424010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:04,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:04,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:04,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:04,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:04,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831424011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:04,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:04,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:04,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:04,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:04,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:04,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:04,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:04,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831424012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:04,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:04,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:04,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:04,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:04,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831424134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:04,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:04,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:04,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:04,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831424134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:04,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:04,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:04,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:04,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:04,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:04,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831424134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:04,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:04,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:04,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:04,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:04,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831424146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:04,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:04,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:04,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:04,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T08:16:04,235 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:16:04,235 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:16:04,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-17T08:16:04,237 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:16:04,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T08:16:04,238 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:16:04,238 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:16:04,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T08:16:04,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:04,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831424355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:04,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:04,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:04,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:04,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:04,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831424355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:04,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:04,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:04,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:04,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:04,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831424359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:04,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:04,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:04,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:04,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:04,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831424363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:04,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:04,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:04,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:04,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T08:16:04,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:04,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:16:04,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:04,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:04,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:04,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:04,406 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=348, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/549a17f8fb734590bd54ae7cbc63f3ab 2024-11-17T08:16:04,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/617a8b1d87da44f79bc2972745708b2b is 50, key is test_row_0/B:col10/1731831363594/Put/seqid=0 2024-11-17T08:16:04,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742214_1390 (size=12301) 2024-11-17T08:16:04,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T08:16:04,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:04,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:16:04,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:04,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:04,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:04,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:04,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T08:16:04,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:04,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831424674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:04,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:04,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:04,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:04,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831424674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:04,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:04,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:04,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:04,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:04,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:04,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831424683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:04,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:04,691 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:04,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831424683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:04,691 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:04,691 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:04,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:04,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:04,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:04,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T08:16:04,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:04,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:16:04,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:04,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:04,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:04,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:04,820 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/617a8b1d87da44f79bc2972745708b2b 2024-11-17T08:16:04,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/21a5f1ce5e764e6591fa809f96ba90a8 is 50, key is test_row_0/C:col10/1731831363594/Put/seqid=0 2024-11-17T08:16:04,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742215_1391 (size=12301) 2024-11-17T08:16:04,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T08:16:04,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:04,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:16:04,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:04,851 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:04,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:04,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:04,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T08:16:05,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T08:16:05,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:05,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:16:05,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:05,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:05,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:05,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:05,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T08:16:05,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:05,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:16:05,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:05,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:05,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:05,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:05,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:05,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831425193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:05,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:05,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:05,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:05,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831425193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:05,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:05,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:05,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:05,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:05,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:05,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831425193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:05,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:05,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:05,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:05,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:05,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831425204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:05,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:05,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:05,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:05,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/21a5f1ce5e764e6591fa809f96ba90a8 2024-11-17T08:16:05,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/549a17f8fb734590bd54ae7cbc63f3ab as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/549a17f8fb734590bd54ae7cbc63f3ab 2024-11-17T08:16:05,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/549a17f8fb734590bd54ae7cbc63f3ab, entries=200, sequenceid=348, filesize=39.0 K 2024-11-17T08:16:05,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/617a8b1d87da44f79bc2972745708b2b as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/617a8b1d87da44f79bc2972745708b2b 2024-11-17T08:16:05,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/617a8b1d87da44f79bc2972745708b2b, entries=150, sequenceid=348, filesize=12.0 K 2024-11-17T08:16:05,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/21a5f1ce5e764e6591fa809f96ba90a8 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/21a5f1ce5e764e6591fa809f96ba90a8 2024-11-17T08:16:05,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/21a5f1ce5e764e6591fa809f96ba90a8, entries=150, sequenceid=348, filesize=12.0 K 2024-11-17T08:16:05,246 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for b852432b87998249586edaadc388ac84 in 1312ms, sequenceid=348, compaction requested=true 2024-11-17T08:16:05,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b852432b87998249586edaadc388ac84: 2024-11-17T08:16:05,246 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-17T08:16:05,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b852432b87998249586edaadc388ac84:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:16:05,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:05,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b852432b87998249586edaadc388ac84:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:16:05,246 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-17T08:16:05,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:05,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b852432b87998249586edaadc388ac84:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:16:05,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:16:05,248 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61881 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-17T08:16:05,248 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): b852432b87998249586edaadc388ac84/B is initiating minor compaction (all files) 2024-11-17T08:16:05,248 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b852432b87998249586edaadc388ac84/B in TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:05,248 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/9cf4d97e5beb49629fd5bdacb34ce11a, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/259cc73715da44279dc50522a99ce362, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/e8e89fd14b534b168ed4dfb3f0908d4b, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/f3550537109449469a45ec194c42e48e, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/617a8b1d87da44f79bc2972745708b2b] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp, totalSize=60.4 K 2024-11-17T08:16:05,248 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 174039 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-17T08:16:05,248 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): b852432b87998249586edaadc388ac84/A is initiating minor compaction (all files) 2024-11-17T08:16:05,248 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b852432b87998249586edaadc388ac84/A in TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:05,248 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/10778409e00a4799a2c99c00a0eb3b8b, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/9b201c52a1b3495e9e6855cdf2d626eb, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/f5519ad431ce49d9843ef756ee056435, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/733c378e5c0946bd91d2c23618ff5e98, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/549a17f8fb734590bd54ae7cbc63f3ab] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp, totalSize=170.0 K 2024-11-17T08:16:05,248 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:05,248 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. files: [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/10778409e00a4799a2c99c00a0eb3b8b, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/9b201c52a1b3495e9e6855cdf2d626eb, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/f5519ad431ce49d9843ef756ee056435, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/733c378e5c0946bd91d2c23618ff5e98, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/549a17f8fb734590bd54ae7cbc63f3ab] 2024-11-17T08:16:05,249 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cf4d97e5beb49629fd5bdacb34ce11a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731831358442 2024-11-17T08:16:05,249 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10778409e00a4799a2c99c00a0eb3b8b, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731831358442 2024-11-17T08:16:05,249 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 259cc73715da44279dc50522a99ce362, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731831359477 2024-11-17T08:16:05,249 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b201c52a1b3495e9e6855cdf2d626eb, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731831359477 2024-11-17T08:16:05,249 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting e8e89fd14b534b168ed4dfb3f0908d4b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731831360664 2024-11-17T08:16:05,249 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5519ad431ce49d9843ef756ee056435, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731831360664 2024-11-17T08:16:05,249 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting f3550537109449469a45ec194c42e48e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731831361359 2024-11-17T08:16:05,250 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 733c378e5c0946bd91d2c23618ff5e98, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731831361359 2024-11-17T08:16:05,250 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 617a8b1d87da44f79bc2972745708b2b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731831363593 2024-11-17T08:16:05,250 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 549a17f8fb734590bd54ae7cbc63f3ab, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731831363575 2024-11-17T08:16:05,256 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:16:05,258 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111783a2c65c0d734d8ea54a1d55364eee73_b852432b87998249586edaadc388ac84 store=[table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:16:05,261 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b852432b87998249586edaadc388ac84#B#compaction#331 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:05,261 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111783a2c65c0d734d8ea54a1d55364eee73_b852432b87998249586edaadc388ac84, store=[table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:16:05,261 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/68ce81ee75c849b59e78362a2e79d8cd is 50, key is test_row_0/B:col10/1731831363594/Put/seqid=0 2024-11-17T08:16:05,261 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111783a2c65c0d734d8ea54a1d55364eee73_b852432b87998249586edaadc388ac84 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:16:05,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742217_1393 (size=12847) 2024-11-17T08:16:05,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742216_1392 (size=4469) 2024-11-17T08:16:05,271 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b852432b87998249586edaadc388ac84#A#compaction#330 average throughput is 1.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:05,272 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/bbeca6763bc7417e8785b3674f69b8f2 is 175, key is test_row_0/A:col10/1731831363594/Put/seqid=0 2024-11-17T08:16:05,275 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/68ce81ee75c849b59e78362a2e79d8cd as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/68ce81ee75c849b59e78362a2e79d8cd 2024-11-17T08:16:05,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742218_1394 (size=31801) 2024-11-17T08:16:05,280 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in b852432b87998249586edaadc388ac84/B of b852432b87998249586edaadc388ac84 into 68ce81ee75c849b59e78362a2e79d8cd(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:05,280 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b852432b87998249586edaadc388ac84: 2024-11-17T08:16:05,280 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., storeName=b852432b87998249586edaadc388ac84/B, priority=11, startTime=1731831365246; duration=0sec 2024-11-17T08:16:05,280 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:16:05,280 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b852432b87998249586edaadc388ac84:B 2024-11-17T08:16:05,280 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-17T08:16:05,282 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61881 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-17T08:16:05,282 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): b852432b87998249586edaadc388ac84/C is initiating minor compaction (all files) 2024-11-17T08:16:05,282 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b852432b87998249586edaadc388ac84/C in TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:05,282 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/ddbd2a788abd49bfabf7beb82de1793e, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/289da5810f6346a3939eead8866dde14, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/820d985e048d4130b3f3507147c1861d, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/db11d2e71b6e476687e6d0d2aaae25c5, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/21a5f1ce5e764e6591fa809f96ba90a8] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp, totalSize=60.4 K 2024-11-17T08:16:05,283 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting ddbd2a788abd49bfabf7beb82de1793e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731831358442 2024-11-17T08:16:05,283 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 289da5810f6346a3939eead8866dde14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731831359477 2024-11-17T08:16:05,283 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 820d985e048d4130b3f3507147c1861d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731831360664 2024-11-17T08:16:05,283 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting db11d2e71b6e476687e6d0d2aaae25c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731831361359 2024-11-17T08:16:05,283 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 21a5f1ce5e764e6591fa809f96ba90a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731831363593 2024-11-17T08:16:05,290 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b852432b87998249586edaadc388ac84#C#compaction#332 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:05,291 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/c0384260db4f42469aedafb0b2d74ee0 is 50, key is test_row_0/C:col10/1731831363594/Put/seqid=0 2024-11-17T08:16:05,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742219_1395 (size=12847) 2024-11-17T08:16:05,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T08:16:05,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:05,309 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing b852432b87998249586edaadc388ac84 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-17T08:16:05,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=A 2024-11-17T08:16:05,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:05,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=B 2024-11-17T08:16:05,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:05,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=C 2024-11-17T08:16:05,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:05,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d112c3dd1c6c47bf82bff3e6febf1f35_b852432b87998249586edaadc388ac84 is 50, key is test_row_0/A:col10/1731831364006/Put/seqid=0 2024-11-17T08:16:05,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742220_1396 (size=12454) 2024-11-17T08:16:05,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:05,331 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d112c3dd1c6c47bf82bff3e6febf1f35_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d112c3dd1c6c47bf82bff3e6febf1f35_b852432b87998249586edaadc388ac84 2024-11-17T08:16:05,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/0fbff8ee7a644b49b4eb7e600163170d, store: [table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:16:05,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/0fbff8ee7a644b49b4eb7e600163170d is 175, key is test_row_0/A:col10/1731831364006/Put/seqid=0 2024-11-17T08:16:05,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742221_1397 (size=31255) 2024-11-17T08:16:05,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T08:16:05,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on b852432b87998249586edaadc388ac84 2024-11-17T08:16:05,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:16:05,680 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/bbeca6763bc7417e8785b3674f69b8f2 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/bbeca6763bc7417e8785b3674f69b8f2 2024-11-17T08:16:05,689 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in b852432b87998249586edaadc388ac84/A of b852432b87998249586edaadc388ac84 into bbeca6763bc7417e8785b3674f69b8f2(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:05,690 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b852432b87998249586edaadc388ac84: 2024-11-17T08:16:05,690 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., storeName=b852432b87998249586edaadc388ac84/A, priority=11, startTime=1731831365246; duration=0sec 2024-11-17T08:16:05,690 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:05,690 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b852432b87998249586edaadc388ac84:A 2024-11-17T08:16:05,702 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/c0384260db4f42469aedafb0b2d74ee0 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/c0384260db4f42469aedafb0b2d74ee0 2024-11-17T08:16:05,706 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in b852432b87998249586edaadc388ac84/C of b852432b87998249586edaadc388ac84 into c0384260db4f42469aedafb0b2d74ee0(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:05,706 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b852432b87998249586edaadc388ac84: 2024-11-17T08:16:05,707 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., storeName=b852432b87998249586edaadc388ac84/C, priority=11, startTime=1731831365247; duration=0sec 2024-11-17T08:16:05,707 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:05,707 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b852432b87998249586edaadc388ac84:C 2024-11-17T08:16:05,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53496 deadline: 1731831425711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:05,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:05,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:05,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:05,739 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=368, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/0fbff8ee7a644b49b4eb7e600163170d 2024-11-17T08:16:05,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/9890ff7173a14b548da6ef4172bd003d is 50, key is test_row_0/B:col10/1731831364006/Put/seqid=0 2024-11-17T08:16:05,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742222_1398 (size=12301) 2024-11-17T08:16:05,758 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/9890ff7173a14b548da6ef4172bd003d 2024-11-17T08:16:05,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/b6d72549b3fc4c9582bb285b26e4847a is 50, key is test_row_0/C:col10/1731831364006/Put/seqid=0 2024-11-17T08:16:05,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742223_1399 (size=12301) 2024-11-17T08:16:05,791 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/b6d72549b3fc4c9582bb285b26e4847a 2024-11-17T08:16:05,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/0fbff8ee7a644b49b4eb7e600163170d as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/0fbff8ee7a644b49b4eb7e600163170d 2024-11-17T08:16:05,802 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/0fbff8ee7a644b49b4eb7e600163170d, entries=150, sequenceid=368, filesize=30.5 K 2024-11-17T08:16:05,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/9890ff7173a14b548da6ef4172bd003d as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/9890ff7173a14b548da6ef4172bd003d 2024-11-17T08:16:05,809 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/9890ff7173a14b548da6ef4172bd003d, entries=150, sequenceid=368, filesize=12.0 K 2024-11-17T08:16:05,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/b6d72549b3fc4c9582bb285b26e4847a as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/b6d72549b3fc4c9582bb285b26e4847a 2024-11-17T08:16:05,816 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/b6d72549b3fc4c9582bb285b26e4847a, entries=150, sequenceid=368, filesize=12.0 K 2024-11-17T08:16:05,817 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for b852432b87998249586edaadc388ac84 in 508ms, sequenceid=368, compaction requested=false 2024-11-17T08:16:05,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for b852432b87998249586edaadc388ac84: 2024-11-17T08:16:05,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:05,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-17T08:16:05,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-17T08:16:05,821 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-17T08:16:05,821 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5800 sec 2024-11-17T08:16:05,823 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.5870 sec 2024-11-17T08:16:05,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on b852432b87998249586edaadc388ac84 2024-11-17T08:16:05,829 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b852432b87998249586edaadc388ac84 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-17T08:16:05,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=A 2024-11-17T08:16:05,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:05,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=B 2024-11-17T08:16:05,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:05,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=C 2024-11-17T08:16:05,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:05,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117cf24958beb4c42fc8d329318a2c65ea1_b852432b87998249586edaadc388ac84 is 50, key is test_row_0/A:col10/1731831365695/Put/seqid=0 2024-11-17T08:16:05,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742224_1400 (size=14994) 2024-11-17T08:16:05,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53496 deadline: 1731831425959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:05,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:05,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:05,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:06,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53496 deadline: 1731831426074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:06,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:06,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:06,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:06,111 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T08:16:06,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:06,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831426214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:06,221 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to e3d0adb87c01,40631,1731831247805 for 'test_row_2' in b852432b87998249586edaadc388ac84 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T08:16:06,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:06,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:06,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:06,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:06,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831426224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:06,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831426224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:06,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831426224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:06,233 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to e3d0adb87c01,40631,1731831247805 for 'test_row_0' in b852432b87998249586edaadc388ac84 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T08:16:06,233 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e3d0adb87c01,40631,1731831247805 for 'test_row_2' in b852432b87998249586edaadc388ac84 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T08:16:06,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:06,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:06,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:06,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:06,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:06,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:06,233 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to e3d0adb87c01,40631,1731831247805 for 'test_row_1' in b852432b87998249586edaadc388ac84 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T08:16:06,234 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:06,234 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:06,234 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:06,274 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:06,277 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117cf24958beb4c42fc8d329318a2c65ea1_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117cf24958beb4c42fc8d329318a2c65ea1_b852432b87998249586edaadc388ac84 2024-11-17T08:16:06,279 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/7946fc3f70e548128e52e71f83751aad, store: [table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:16:06,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/7946fc3f70e548128e52e71f83751aad is 175, key is test_row_0/A:col10/1731831365695/Put/seqid=0 2024-11-17T08:16:06,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53496 deadline: 1731831426284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:06,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742225_1401 (size=39949) 2024-11-17T08:16:06,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:06,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:06,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T08:16:06,384 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:16:06,384 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:16:06,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-17T08:16:06,386 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:16:06,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T08:16:06,386 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:16:06,386 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:16:06,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T08:16:06,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T08:16:06,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:06,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:16:06,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:06,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:06,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:06,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:06,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:06,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53496 deadline: 1731831426603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:06,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:06,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:06,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:06,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T08:16:06,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:06,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:16:06,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:06,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:06,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:06,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:06,693 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=384, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/7946fc3f70e548128e52e71f83751aad 2024-11-17T08:16:06,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/39927293fcbf4461b42859f6940443fd is 50, key is test_row_0/B:col10/1731831365695/Put/seqid=0 2024-11-17T08:16:06,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742226_1402 (size=12301) 2024-11-17T08:16:06,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T08:16:06,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T08:16:06,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:06,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:16:06,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:06,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:06,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:06,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:06,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T08:16:06,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:06,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:16:06,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:06,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:06,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:06,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:07,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T08:16:07,102 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/39927293fcbf4461b42859f6940443fd 2024-11-17T08:16:07,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/9de13eb229ef4cdeab936b50c177be9c is 50, key is test_row_0/C:col10/1731831365695/Put/seqid=0 2024-11-17T08:16:07,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742227_1403 (size=12301) 2024-11-17T08:16:07,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/9de13eb229ef4cdeab936b50c177be9c 2024-11-17T08:16:07,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/7946fc3f70e548128e52e71f83751aad as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/7946fc3f70e548128e52e71f83751aad 2024-11-17T08:16:07,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/7946fc3f70e548128e52e71f83751aad, entries=200, sequenceid=384, filesize=39.0 K 2024-11-17T08:16:07,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/39927293fcbf4461b42859f6940443fd as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/39927293fcbf4461b42859f6940443fd 2024-11-17T08:16:07,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/39927293fcbf4461b42859f6940443fd, entries=150, sequenceid=384, filesize=12.0 K 2024-11-17T08:16:07,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/9de13eb229ef4cdeab936b50c177be9c as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/9de13eb229ef4cdeab936b50c177be9c 2024-11-17T08:16:07,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:07,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53496 deadline: 1731831427124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:07,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:07,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:07,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:07,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/9de13eb229ef4cdeab936b50c177be9c, entries=150, sequenceid=384, filesize=12.0 K 2024-11-17T08:16:07,134 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for b852432b87998249586edaadc388ac84 in 1305ms, sequenceid=384, compaction requested=true 2024-11-17T08:16:07,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b852432b87998249586edaadc388ac84: 2024-11-17T08:16:07,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b852432b87998249586edaadc388ac84:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:16:07,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:07,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b852432b87998249586edaadc388ac84:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:16:07,134 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:16:07,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:07,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b852432b87998249586edaadc388ac84:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:16:07,134 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:16:07,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:16:07,135 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:16:07,135 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): b852432b87998249586edaadc388ac84/B is initiating minor compaction (all files) 2024-11-17T08:16:07,135 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:16:07,135 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b852432b87998249586edaadc388ac84/B in TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:07,135 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): b852432b87998249586edaadc388ac84/A is initiating minor compaction (all files) 2024-11-17T08:16:07,135 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b852432b87998249586edaadc388ac84/A in TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:07,135 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/68ce81ee75c849b59e78362a2e79d8cd, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/9890ff7173a14b548da6ef4172bd003d, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/39927293fcbf4461b42859f6940443fd] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp, totalSize=36.6 K 2024-11-17T08:16:07,135 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/bbeca6763bc7417e8785b3674f69b8f2, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/0fbff8ee7a644b49b4eb7e600163170d, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/7946fc3f70e548128e52e71f83751aad] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp, totalSize=100.6 K 2024-11-17T08:16:07,135 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:07,135 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. files: [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/bbeca6763bc7417e8785b3674f69b8f2, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/0fbff8ee7a644b49b4eb7e600163170d, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/7946fc3f70e548128e52e71f83751aad] 2024-11-17T08:16:07,135 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 68ce81ee75c849b59e78362a2e79d8cd, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731831363593 2024-11-17T08:16:07,135 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbeca6763bc7417e8785b3674f69b8f2, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731831363593 2024-11-17T08:16:07,136 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 9890ff7173a14b548da6ef4172bd003d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731831364006 2024-11-17T08:16:07,136 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fbff8ee7a644b49b4eb7e600163170d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731831364006 2024-11-17T08:16:07,136 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 39927293fcbf4461b42859f6940443fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731831365695 2024-11-17T08:16:07,136 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7946fc3f70e548128e52e71f83751aad, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731831365649 2024-11-17T08:16:07,146 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b852432b87998249586edaadc388ac84#B#compaction#339 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:07,147 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/14de48c9035c4ca6aa52e68ed532a508 is 50, key is test_row_0/B:col10/1731831365695/Put/seqid=0 2024-11-17T08:16:07,150 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:16:07,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T08:16:07,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:07,152 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing b852432b87998249586edaadc388ac84 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-17T08:16:07,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=A 2024-11-17T08:16:07,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:07,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=B 2024-11-17T08:16:07,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:07,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=C 2024-11-17T08:16:07,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:07,161 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411174f8484db4cf74772a56f9c37dddd3958_b852432b87998249586edaadc388ac84 store=[table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:16:07,162 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411174f8484db4cf74772a56f9c37dddd3958_b852432b87998249586edaadc388ac84, store=[table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:16:07,162 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174f8484db4cf74772a56f9c37dddd3958_b852432b87998249586edaadc388ac84 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:16:07,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742228_1404 (size=12949) 2024-11-17T08:16:07,170 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/14de48c9035c4ca6aa52e68ed532a508 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/14de48c9035c4ca6aa52e68ed532a508 2024-11-17T08:16:07,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411176f898a54b0764fddb494d630d339d30a_b852432b87998249586edaadc388ac84 is 50, key is test_row_0/A:col10/1731831365958/Put/seqid=0 2024-11-17T08:16:07,175 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b852432b87998249586edaadc388ac84/B of b852432b87998249586edaadc388ac84 into 14de48c9035c4ca6aa52e68ed532a508(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:07,175 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b852432b87998249586edaadc388ac84: 2024-11-17T08:16:07,175 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., storeName=b852432b87998249586edaadc388ac84/B, priority=13, startTime=1731831367134; duration=0sec 2024-11-17T08:16:07,175 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:16:07,175 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b852432b87998249586edaadc388ac84:B 2024-11-17T08:16:07,175 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:16:07,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742229_1405 (size=4469) 2024-11-17T08:16:07,188 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b852432b87998249586edaadc388ac84#A#compaction#340 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:07,188 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:16:07,188 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): b852432b87998249586edaadc388ac84/C is initiating minor compaction (all files) 2024-11-17T08:16:07,188 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b852432b87998249586edaadc388ac84/C in TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:07,188 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/c0384260db4f42469aedafb0b2d74ee0, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/b6d72549b3fc4c9582bb285b26e4847a, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/9de13eb229ef4cdeab936b50c177be9c] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp, totalSize=36.6 K 2024-11-17T08:16:07,188 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/e05224a12920486b98eec06c4e53f1dc is 175, key is test_row_0/A:col10/1731831365695/Put/seqid=0 2024-11-17T08:16:07,189 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting c0384260db4f42469aedafb0b2d74ee0, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731831363593 2024-11-17T08:16:07,189 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting b6d72549b3fc4c9582bb285b26e4847a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731831364006 2024-11-17T08:16:07,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742230_1406 (size=12454) 2024-11-17T08:16:07,190 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 9de13eb229ef4cdeab936b50c177be9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731831365695 2024-11-17T08:16:07,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:07,195 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411176f898a54b0764fddb494d630d339d30a_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176f898a54b0764fddb494d630d339d30a_b852432b87998249586edaadc388ac84 2024-11-17T08:16:07,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/3728bbc0a4084d99b757420f0bc82c9f, store: [table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:16:07,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/3728bbc0a4084d99b757420f0bc82c9f is 175, key is test_row_0/A:col10/1731831365958/Put/seqid=0 2024-11-17T08:16:07,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742231_1407 (size=31903) 2024-11-17T08:16:07,199 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b852432b87998249586edaadc388ac84#C#compaction#342 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:07,199 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/b66925eea51946d1b84b1950e629829b is 50, key is test_row_0/C:col10/1731831365695/Put/seqid=0 2024-11-17T08:16:07,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742232_1408 (size=31255) 2024-11-17T08:16:07,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742233_1409 (size=12949) 2024-11-17T08:16:07,231 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/b66925eea51946d1b84b1950e629829b as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/b66925eea51946d1b84b1950e629829b 2024-11-17T08:16:07,237 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b852432b87998249586edaadc388ac84/C of b852432b87998249586edaadc388ac84 into b66925eea51946d1b84b1950e629829b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:07,237 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b852432b87998249586edaadc388ac84: 2024-11-17T08:16:07,237 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., storeName=b852432b87998249586edaadc388ac84/C, priority=13, startTime=1731831367134; duration=0sec 2024-11-17T08:16:07,237 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:07,237 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b852432b87998249586edaadc388ac84:C 2024-11-17T08:16:07,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T08:16:07,603 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/e05224a12920486b98eec06c4e53f1dc as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/e05224a12920486b98eec06c4e53f1dc 2024-11-17T08:16:07,606 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b852432b87998249586edaadc388ac84/A of b852432b87998249586edaadc388ac84 into e05224a12920486b98eec06c4e53f1dc(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:07,607 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b852432b87998249586edaadc388ac84: 2024-11-17T08:16:07,607 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., storeName=b852432b87998249586edaadc388ac84/A, priority=13, startTime=1731831367134; duration=0sec 2024-11-17T08:16:07,607 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:07,607 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b852432b87998249586edaadc388ac84:A 2024-11-17T08:16:07,610 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=402, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/3728bbc0a4084d99b757420f0bc82c9f 2024-11-17T08:16:07,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/d327c6d8f5d14251bae62de9b53b159d is 50, key is test_row_0/B:col10/1731831365958/Put/seqid=0 2024-11-17T08:16:07,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742234_1410 (size=12301) 2024-11-17T08:16:08,018 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/d327c6d8f5d14251bae62de9b53b159d 2024-11-17T08:16:08,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/bcef6d5ec73140a68e2866cfa494e835 is 50, key is test_row_0/C:col10/1731831365958/Put/seqid=0 2024-11-17T08:16:08,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742235_1411 (size=12301) 2024-11-17T08:16:08,058 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/bcef6d5ec73140a68e2866cfa494e835 2024-11-17T08:16:08,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/3728bbc0a4084d99b757420f0bc82c9f as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/3728bbc0a4084d99b757420f0bc82c9f 2024-11-17T08:16:08,068 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/3728bbc0a4084d99b757420f0bc82c9f, entries=150, sequenceid=402, filesize=30.5 K 2024-11-17T08:16:08,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/d327c6d8f5d14251bae62de9b53b159d as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/d327c6d8f5d14251bae62de9b53b159d 2024-11-17T08:16:08,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,072 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/d327c6d8f5d14251bae62de9b53b159d, entries=150, sequenceid=402, filesize=12.0 K 2024-11-17T08:16:08,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/bcef6d5ec73140a68e2866cfa494e835 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/bcef6d5ec73140a68e2866cfa494e835 2024-11-17T08:16:08,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,078 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/bcef6d5ec73140a68e2866cfa494e835, entries=150, sequenceid=402, filesize=12.0 K 2024-11-17T08:16:08,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,079 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for b852432b87998249586edaadc388ac84 in 927ms, sequenceid=402, compaction requested=false 2024-11-17T08:16:08,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for b852432b87998249586edaadc388ac84: 2024-11-17T08:16:08,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:08,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-17T08:16:08,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-17T08:16:08,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,082 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-17T08:16:08,082 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6940 sec 2024-11-17T08:16:08,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,084 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.6990 sec 2024-11-17T08:16:08,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on b852432b87998249586edaadc388ac84 2024-11-17T08:16:08,273 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b852432b87998249586edaadc388ac84 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T08:16:08,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=A 2024-11-17T08:16:08,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:08,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=B 2024-11-17T08:16:08,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:08,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=C 2024-11-17T08:16:08,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:08,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411179ee3f56dc01449fdbbdb0aa33a2d26f4_b852432b87998249586edaadc388ac84 is 50, key is test_row_0/A:col10/1731831368261/Put/seqid=0 2024-11-17T08:16:08,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742236_1412 (size=22618) 2024-11-17T08:16:08,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:08,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831428321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:08,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:08,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:08,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:08,348 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731831368318, max timestamp=1731831368345Num uniques 859; 2024-11-17T08:16:08,349 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T08:16:08,349 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731831368318, max timestamp=1731831368345Num uniques 859; 2024-11-17T08:16:08,349 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T08:16:08,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731831368318, max timestamp=1731831368345Num uniques 859; 2024-11-17T08:16:08,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T08:16:08,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:08,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53496 deadline: 1731831428393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:08,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:08,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:08,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831428398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:08,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:08,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:08,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:08,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:08,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:08,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:08,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831428402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:08,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:08,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:08,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:08,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:08,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831428403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:08,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:08,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:08,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:08,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:08,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831428454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:08,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:08,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:08,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:08,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:08,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53496 deadline: 1731831428513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:08,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:08,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:08,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:08,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:08,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831428514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:08,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:08,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:08,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:08,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:08,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831428524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:08,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:08,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:08,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831428524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:08,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:08,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:08,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:08,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:08,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:08,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T08:16:08,543 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:16:08,544 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:16:08,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-17T08:16:08,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T08:16:08,546 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:16:08,546 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:16:08,546 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:16:08,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T08:16:08,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:08,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831428673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:08,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:08,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:08,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:08,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T08:16:08,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:08,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:16:08,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:08,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:08,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:08,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:08,718 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:08,722 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411179ee3f56dc01449fdbbdb0aa33a2d26f4_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411179ee3f56dc01449fdbbdb0aa33a2d26f4_b852432b87998249586edaadc388ac84 2024-11-17T08:16:08,722 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/ce01e64c9d124a94a87d66fda7728a0d, store: [table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:16:08,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/ce01e64c9d124a94a87d66fda7728a0d is 175, key is test_row_0/A:col10/1731831368261/Put/seqid=0 2024-11-17T08:16:08,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742237_1413 (size=66023) 2024-11-17T08:16:08,726 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=415, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/ce01e64c9d124a94a87d66fda7728a0d 2024-11-17T08:16:08,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/86a2452142524f60a25f6e01d4b4b507 is 50, key is test_row_0/B:col10/1731831368261/Put/seqid=0 2024-11-17T08:16:08,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:08,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53496 deadline: 1731831428724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:08,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:08,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831428724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:08,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:08,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:08,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:08,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:08,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:08,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:08,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742238_1414 (size=12301) 2024-11-17T08:16:08,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:08,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831428743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:08,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:08,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:08,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831428743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:08,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:08,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:08,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:08,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:08,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:08,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T08:16:08,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:08,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:16:08,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:08,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:08,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:08,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:08,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T08:16:08,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:08,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831428994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:08,997 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:08,997 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:08,997 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:09,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T08:16:09,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:09,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:16:09,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:09,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:09,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:09,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:09,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:09,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53496 deadline: 1731831429043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:09,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:09,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:09,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:09,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831429043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:09,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:09,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:09,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:09,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:09,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:09,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831429064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:09,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:09,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:09,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:09,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831429064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:09,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:09,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:09,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:09,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:09,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/86a2452142524f60a25f6e01d4b4b507 2024-11-17T08:16:09,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/1a0d82e32f7d405d865ced75ad3f778a is 50, key is test_row_0/C:col10/1731831368261/Put/seqid=0 2024-11-17T08:16:09,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742239_1415 (size=12301) 2024-11-17T08:16:09,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T08:16:09,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:09,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:16:09,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:09,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:09,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:09,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:09,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T08:16:09,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T08:16:09,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:09,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:16:09,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:09,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:09,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:09,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T08:16:09,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:09,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:16:09,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:09,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:09,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:09,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:09,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:09,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831429503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:09,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:09,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:09,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:09,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/1a0d82e32f7d405d865ced75ad3f778a 2024-11-17T08:16:09,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/ce01e64c9d124a94a87d66fda7728a0d as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/ce01e64c9d124a94a87d66fda7728a0d 2024-11-17T08:16:09,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/ce01e64c9d124a94a87d66fda7728a0d, entries=350, sequenceid=415, filesize=64.5 K 2024-11-17T08:16:09,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/86a2452142524f60a25f6e01d4b4b507 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/86a2452142524f60a25f6e01d4b4b507 2024-11-17T08:16:09,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/86a2452142524f60a25f6e01d4b4b507, entries=150, sequenceid=415, filesize=12.0 K 2024-11-17T08:16:09,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/1a0d82e32f7d405d865ced75ad3f778a as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/1a0d82e32f7d405d865ced75ad3f778a 2024-11-17T08:16:09,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:09,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53496 deadline: 1731831429554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:09,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/1a0d82e32f7d405d865ced75ad3f778a, entries=150, sequenceid=415, filesize=12.0 K 2024-11-17T08:16:09,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:09,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:09,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:09,566 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for b852432b87998249586edaadc388ac84 in 1294ms, sequenceid=415, compaction requested=true 2024-11-17T08:16:09,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b852432b87998249586edaadc388ac84: 2024-11-17T08:16:09,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b852432b87998249586edaadc388ac84:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:16:09,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:09,566 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:16:09,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b852432b87998249586edaadc388ac84:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:16:09,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:09,566 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:16:09,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b852432b87998249586edaadc388ac84:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:16:09,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:16:09,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on b852432b87998249586edaadc388ac84 2024-11-17T08:16:09,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731831368397, max timestamp=1731831369565Num uniques 1018; 2024-11-17T08:16:09,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-17T08:16:09,569 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b852432b87998249586edaadc388ac84 3/3 column families, dataSize=102.87 KB heapSize=292.20 KB 2024-11-17T08:16:09,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731831368397, max timestamp=1731831369565Num uniques 1018; 2024-11-17T08:16:09,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=A 2024-11-17T08:16:09,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731831368397, max timestamp=1731831369565Num uniques 1018; 2024-11-17T08:16:09,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:16:09,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=B 2024-11-17T08:16:09,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-17T08:16:09,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:16:09,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-17T08:16:09,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=C 2024-11-17T08:16:09,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:16:09,570 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:16:09,570 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 129181 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:16:09,570 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): b852432b87998249586edaadc388ac84/B is initiating minor compaction (all files) 2024-11-17T08:16:09,570 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): b852432b87998249586edaadc388ac84/A is initiating minor compaction (all files) 2024-11-17T08:16:09,570 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b852432b87998249586edaadc388ac84/A in TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:09,570 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b852432b87998249586edaadc388ac84/B in TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:09,570 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/14de48c9035c4ca6aa52e68ed532a508, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/d327c6d8f5d14251bae62de9b53b159d, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/86a2452142524f60a25f6e01d4b4b507] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp, totalSize=36.7 K 2024-11-17T08:16:09,570 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/e05224a12920486b98eec06c4e53f1dc, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/3728bbc0a4084d99b757420f0bc82c9f, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/ce01e64c9d124a94a87d66fda7728a0d] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp, totalSize=126.2 K 2024-11-17T08:16:09,570 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:09,570 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. files: [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/e05224a12920486b98eec06c4e53f1dc, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/3728bbc0a4084d99b757420f0bc82c9f, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/ce01e64c9d124a94a87d66fda7728a0d] 2024-11-17T08:16:09,571 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 14de48c9035c4ca6aa52e68ed532a508, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731831365695 2024-11-17T08:16:09,571 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting e05224a12920486b98eec06c4e53f1dc, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731831365695 2024-11-17T08:16:09,571 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3728bbc0a4084d99b757420f0bc82c9f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731831365892 2024-11-17T08:16:09,571 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting d327c6d8f5d14251bae62de9b53b159d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731831365892 2024-11-17T08:16:09,572 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 86a2452142524f60a25f6e01d4b4b507, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731831368261 2024-11-17T08:16:09,572 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce01e64c9d124a94a87d66fda7728a0d, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731831368205 2024-11-17T08:16:09,577 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:16:09,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ec01eab81ada451692b7bdc32ac2195f_b852432b87998249586edaadc388ac84 is 50, key is test_row_0/A:col10/1731831368397/Put/seqid=0 2024-11-17T08:16:09,579 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b852432b87998249586edaadc388ac84#B#compaction#350 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:09,580 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/94956228aab84fafbcc13c5ebfdbd8ef is 50, key is test_row_0/B:col10/1731831368261/Put/seqid=0 2024-11-17T08:16:09,580 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117d9eea4f5b13a4271bc65f6326b6514e3_b852432b87998249586edaadc388ac84 store=[table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:16:09,582 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117d9eea4f5b13a4271bc65f6326b6514e3_b852432b87998249586edaadc388ac84, store=[table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:16:09,582 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d9eea4f5b13a4271bc65f6326b6514e3_b852432b87998249586edaadc388ac84 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:16:09,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742241_1417 (size=13051) 2024-11-17T08:16:09,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742240_1416 (size=14994) 2024-11-17T08:16:09,586 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:09,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742242_1418 (size=4469) 2024-11-17T08:16:09,589 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ec01eab81ada451692b7bdc32ac2195f_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ec01eab81ada451692b7bdc32ac2195f_b852432b87998249586edaadc388ac84 2024-11-17T08:16:09,589 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/94956228aab84fafbcc13c5ebfdbd8ef as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/94956228aab84fafbcc13c5ebfdbd8ef 2024-11-17T08:16:09,590 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/157be605c56848609887dd0ae3894145, store: [table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:16:09,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/157be605c56848609887dd0ae3894145 is 175, key is test_row_0/A:col10/1731831368397/Put/seqid=0 2024-11-17T08:16:09,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742243_1419 (size=39949) 2024-11-17T08:16:09,595 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b852432b87998249586edaadc388ac84/B of b852432b87998249586edaadc388ac84 into 94956228aab84fafbcc13c5ebfdbd8ef(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:09,595 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b852432b87998249586edaadc388ac84: 2024-11-17T08:16:09,596 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., storeName=b852432b87998249586edaadc388ac84/B, priority=13, startTime=1731831369566; duration=0sec 2024-11-17T08:16:09,596 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:16:09,596 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b852432b87998249586edaadc388ac84:B 2024-11-17T08:16:09,596 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:16:09,596 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:16:09,596 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): b852432b87998249586edaadc388ac84/C is initiating minor compaction (all files) 2024-11-17T08:16:09,596 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b852432b87998249586edaadc388ac84/C in TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:09,596 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/b66925eea51946d1b84b1950e629829b, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/bcef6d5ec73140a68e2866cfa494e835, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/1a0d82e32f7d405d865ced75ad3f778a] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp, totalSize=36.7 K 2024-11-17T08:16:09,597 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting b66925eea51946d1b84b1950e629829b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731831365695 2024-11-17T08:16:09,597 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting bcef6d5ec73140a68e2866cfa494e835, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731831365892 2024-11-17T08:16:09,597 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a0d82e32f7d405d865ced75ad3f778a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731831368261 2024-11-17T08:16:09,606 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b852432b87998249586edaadc388ac84#C#compaction#351 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:09,607 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/0eb0c229b5d04b27aaeb18fa08eba802 is 50, key is test_row_0/C:col10/1731831368261/Put/seqid=0 2024-11-17T08:16:09,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742244_1420 (size=13051) 2024-11-17T08:16:09,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T08:16:09,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:09,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:16:09,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:09,622 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:09,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:09,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:09,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731831369646, max timestamp=1731831369664Num uniques 859; 2024-11-17T08:16:09,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-17T08:16:09,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731831369646, max timestamp=1731831369664Num uniques 859; 2024-11-17T08:16:09,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731831369646, max timestamp=1731831369664Num uniques 859; 2024-11-17T08:16:09,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-17T08:16:09,668 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-17T08:16:09,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T08:16:09,724 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731831369683, max timestamp=1731831369721Num uniques 1168; 2024-11-17T08:16:09,724 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=14558, suffix heap size=149440,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=318 2024-11-17T08:16:09,724 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731831369683, max timestamp=1731831369721Num uniques 1168; 2024-11-17T08:16:09,724 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=14558, suffix heap size=149440,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=318 2024-11-17T08:16:09,724 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731831369683, max timestamp=1731831369721Num uniques 1168; 2024-11-17T08:16:09,724 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=14558, suffix heap size=149440,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=318 2024-11-17T08:16:09,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T08:16:09,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:09,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:16:09,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:09,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:09,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:09,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:09,792 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731831369749, max timestamp=1731831369788Num uniques 1177; 2024-11-17T08:16:09,792 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=12682, suffix heap size=150376,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=277 2024-11-17T08:16:09,792 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731831369749, max timestamp=1731831369788Num uniques 1177; 2024-11-17T08:16:09,792 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=12682, suffix heap size=150376,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=277 2024-11-17T08:16:09,793 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731831369749, max timestamp=1731831369788Num uniques 1177; 2024-11-17T08:16:09,793 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=12682, suffix heap size=150376,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=277 2024-11-17T08:16:09,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:09,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831429849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:09,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:09,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:09,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:09,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731831369823, max timestamp=1731831369850Num uniques 1136; 2024-11-17T08:16:09,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=13096, suffix heap size=146112,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=286 2024-11-17T08:16:09,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731831369823, max timestamp=1731831369850Num uniques 1136; 2024-11-17T08:16:09,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731831369823, max timestamp=1731831369850Num uniques 1136; 2024-11-17T08:16:09,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=13096, suffix heap size=146112,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=286 2024-11-17T08:16:09,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=13096, suffix heap size=146112,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=286 2024-11-17T08:16:09,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T08:16:09,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:09,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:16:09,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:09,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:09,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:09,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:09,943 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731831369905, max timestamp=1731831369939Num uniques 1145; 2024-11-17T08:16:09,943 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=13510, suffix heap size=147048,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=295 2024-11-17T08:16:09,943 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731831369905, max timestamp=1731831369939Num uniques 1145; 2024-11-17T08:16:09,943 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1731831369905, max timestamp=1731831369939Num uniques 1145; 2024-11-17T08:16:09,943 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=13510, suffix heap size=147048,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=295 2024-11-17T08:16:09,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=13510, suffix heap size=147048,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=295 2024-11-17T08:16:09,989 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b852432b87998249586edaadc388ac84#A#compaction#349 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:09,989 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/fc41cefac1fb43e680e11ed755e379f3 is 175, key is test_row_0/A:col10/1731831368261/Put/seqid=0 2024-11-17T08:16:09,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742245_1421 (size=32005) 2024-11-17T08:16:09,995 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/fc41cefac1fb43e680e11ed755e379f3 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/fc41cefac1fb43e680e11ed755e379f3 2024-11-17T08:16:09,996 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=453, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/157be605c56848609887dd0ae3894145 2024-11-17T08:16:09,999 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b852432b87998249586edaadc388ac84/A of b852432b87998249586edaadc388ac84 into fc41cefac1fb43e680e11ed755e379f3(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:09,999 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b852432b87998249586edaadc388ac84: 2024-11-17T08:16:09,999 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., storeName=b852432b87998249586edaadc388ac84/A, priority=13, startTime=1731831369566; duration=0sec 2024-11-17T08:16:09,999 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:09,999 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b852432b87998249586edaadc388ac84:A 2024-11-17T08:16:10,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:10,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831430000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:10,011 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:10,011 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:10,011 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:10,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731831369990, max timestamp=1731831370007Num uniques 1154; 2024-11-17T08:16:10,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731831369990, max timestamp=1731831370007Num uniques 1154; 2024-11-17T08:16:10,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=9340, suffix heap size=147984,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=204 2024-11-17T08:16:10,012 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=9340, suffix heap size=147984,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=204 2024-11-17T08:16:10,012 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731831369990, max timestamp=1731831370007Num uniques 1154; 2024-11-17T08:16:10,012 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=9340, suffix heap size=147984,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=204 2024-11-17T08:16:10,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/0f77caab5cef4ef2b8133e446eba44ae is 50, key is test_row_0/B:col10/1731831368397/Put/seqid=0 2024-11-17T08:16:10,016 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/0eb0c229b5d04b27aaeb18fa08eba802 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/0eb0c229b5d04b27aaeb18fa08eba802 2024-11-17T08:16:10,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742246_1422 (size=12301) 2024-11-17T08:16:10,020 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b852432b87998249586edaadc388ac84/C of b852432b87998249586edaadc388ac84 into 0eb0c229b5d04b27aaeb18fa08eba802(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:10,020 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b852432b87998249586edaadc388ac84: 2024-11-17T08:16:10,020 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., storeName=b852432b87998249586edaadc388ac84/C, priority=13, startTime=1731831369566; duration=0sec 2024-11-17T08:16:10,021 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:10,021 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b852432b87998249586edaadc388ac84:C 2024-11-17T08:16:10,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T08:16:10,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:10,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:16:10,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:10,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:10,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:10,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:10,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731831370068, max timestamp=1731831370081Num uniques 1063; 2024-11-17T08:16:10,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-17T08:16:10,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731831370068, max timestamp=1731831370081Num uniques 1063; 2024-11-17T08:16:10,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731831370068, max timestamp=1731831370081Num uniques 1063; 2024-11-17T08:16:10,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-17T08:16:10,085 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-17T08:16:10,157 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731831370139, max timestamp=1731831370153Num uniques 1022; 2024-11-17T08:16:10,157 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731831370139, max timestamp=1731831370153Num uniques 1022; 2024-11-17T08:16:10,157 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-17T08:16:10,157 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-17T08:16:10,157 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731831370139, max timestamp=1731831370153Num uniques 1022; 2024-11-17T08:16:10,157 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=10162, suffix heap size=134256,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=222 2024-11-17T08:16:10,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T08:16:10,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:10,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:16:10,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:10,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:10,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:10,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:10,254 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731831370235, max timestamp=1731831370251Num uniques 1081; 2024-11-17T08:16:10,254 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-11-17T08:16:10,254 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731831370235, max timestamp=1731831370251Num uniques 1081; 2024-11-17T08:16:10,254 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-11-17T08:16:10,255 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731831370235, max timestamp=1731831370251Num uniques 1081; 2024-11-17T08:16:10,255 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=10576, suffix heap size=140392,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=231 2024-11-17T08:16:10,305 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1731831370287, max timestamp=1731831370301Num uniques 1090; 2024-11-17T08:16:10,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=13280, suffix heap size=141328,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=290 2024-11-17T08:16:10,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1731831370287, max timestamp=1731831370301Num uniques 1090; 2024-11-17T08:16:10,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=13280, suffix heap size=141328,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=290 2024-11-17T08:16:10,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1731831370287, max timestamp=1731831370301Num uniques 1090; 2024-11-17T08:16:10,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=13280, suffix heap size=141328,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=290 2024-11-17T08:16:10,358 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1731831370341, max timestamp=1731831370354Num uniques 1149; 2024-11-17T08:16:10,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52624, new segment data size=13694, suffix heap size=147464,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1149, new segment cells count=299 2024-11-17T08:16:10,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1731831370341, max timestamp=1731831370354Num uniques 1149; 2024-11-17T08:16:10,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52624, new segment data size=13694, suffix heap size=147464,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1149, new segment cells count=299 2024-11-17T08:16:10,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=299, cellSize=13694, totalHeapSize=31336, min timestamp=1731831370341, max timestamp=1731831370354Num uniques 1149; 2024-11-17T08:16:10,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52624, new segment data size=13694, suffix heap size=147464,new segment heap size=31336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1149, new segment cells count=299 2024-11-17T08:16:10,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T08:16:10,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:10,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:16:10,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:10,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:10,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:10,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:10,417 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/0f77caab5cef4ef2b8133e446eba44ae 2024-11-17T08:16:10,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/a06f25ba877f42739f4f0c62e0e3d89c is 50, key is test_row_0/C:col10/1731831368397/Put/seqid=0 2024-11-17T08:16:10,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:10,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831430426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:10,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:10,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831430426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:10,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:10,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:10,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:10,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:10,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:10,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:10,442 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731831370419, max timestamp=1731831370439Num uniques 1158; 2024-11-17T08:16:10,442 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=53030, new segment data size=7230, suffix heap size=148400,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1158, new segment cells count=158 2024-11-17T08:16:10,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731831370419, max timestamp=1731831370439Num uniques 1158; 2024-11-17T08:16:10,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=53030, new segment data size=7230, suffix heap size=148400,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1158, new segment cells count=158 2024-11-17T08:16:10,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731831370419, max timestamp=1731831370439Num uniques 1158; 2024-11-17T08:16:10,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=53030, new segment data size=7230, suffix heap size=148400,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1158, new segment cells count=158 2024-11-17T08:16:10,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742247_1423 (size=12301) 2024-11-17T08:16:10,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/a06f25ba877f42739f4f0c62e0e3d89c 2024-11-17T08:16:10,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/157be605c56848609887dd0ae3894145 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/157be605c56848609887dd0ae3894145 2024-11-17T08:16:10,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/157be605c56848609887dd0ae3894145, entries=200, sequenceid=453, filesize=39.0 K 2024-11-17T08:16:10,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/0f77caab5cef4ef2b8133e446eba44ae as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/0f77caab5cef4ef2b8133e446eba44ae 2024-11-17T08:16:10,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/0f77caab5cef4ef2b8133e446eba44ae, entries=150, sequenceid=453, filesize=12.0 K 2024-11-17T08:16:10,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/a06f25ba877f42739f4f0c62e0e3d89c as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/a06f25ba877f42739f4f0c62e0e3d89c 2024-11-17T08:16:10,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/a06f25ba877f42739f4f0c62e0e3d89c, entries=150, sequenceid=453, filesize=12.0 K 2024-11-17T08:16:10,480 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~65.34 KB/66912, currentSize=60.38 KB/61830 for b852432b87998249586edaadc388ac84 in 911ms, sequenceid=453, compaction requested=false 2024-11-17T08:16:10,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b852432b87998249586edaadc388ac84: 2024-11-17T08:16:10,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on b852432b87998249586edaadc388ac84 2024-11-17T08:16:10,483 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b852432b87998249586edaadc388ac84 3/3 column families, dataSize=67.09 KB heapSize=185.86 KB 2024-11-17T08:16:10,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=A 2024-11-17T08:16:10,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:16:10,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=B 2024-11-17T08:16:10,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:16:10,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=C 2024-11-17T08:16:10,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:16:10,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117047572663e024b6589a220b21a695094_b852432b87998249586edaadc388ac84 is 50, key is test_row_0/A:col10/1731831370476/Put/seqid=0 2024-11-17T08:16:10,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742248_1424 (size=14994) 2024-11-17T08:16:10,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T08:16:10,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:10,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:16:10,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:10,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:10,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:10,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:10,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:10,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831430559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:10,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:10,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:10,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:10,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:10,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831430560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:10,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:10,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:10,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:10,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:10,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831430561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:10,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:10,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:10,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:10,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:10,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831430569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:10,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:10,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:10,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:10,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:10,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53496 deadline: 1731831430574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:10,582 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e3d0adb87c01,40631,1731831247805 for 'test_row_1' in b852432b87998249586edaadc388ac84 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T08:16:10,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:10,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:10,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:10,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:10,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831430674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:10,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:10,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:10,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831430674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:10,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:10,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:10,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:10,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:10,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:10,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:10,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831430684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:10,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:10,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T08:16:10,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831430684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:10,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:10,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:16:10,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:10,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:10,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:10,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:10,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:10,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:10,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:10,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:10,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:10,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:10,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T08:16:10,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T08:16:10,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:10,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:16:10,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:10,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:10,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:10,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:10,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:10,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831430884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:10,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:10,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:10,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831430884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:10,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:10,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:10,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:10,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:10,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:10,896 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:10,898 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117047572663e024b6589a220b21a695094_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117047572663e024b6589a220b21a695094_b852432b87998249586edaadc388ac84 2024-11-17T08:16:10,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:10,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831430893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:10,899 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/a427f1a4533e4ffba1cd0ccc6492ffdb, store: [table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:16:10,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:10,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:10,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:10,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:10,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831430893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:10,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:10,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:10,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:10,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/a427f1a4533e4ffba1cd0ccc6492ffdb is 175, key is test_row_0/A:col10/1731831370476/Put/seqid=0 2024-11-17T08:16:10,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742249_1425 (size=39949) 2024-11-17T08:16:10,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T08:16:10,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:10,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:16:10,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:10,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:10,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:10,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:11,051 WARN [regionserver/e3d0adb87c01:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-17T08:16:11,052 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/b7545be14c1a432f83eeb367c7f7eb9c, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/ffb3418ecacf41ceae841c4e77d1e06f, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/0ba8e8a75621419fa50eb91f5da4edd7, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/86132cf5fbc949b1b31513a745152274, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/f0ce9c39152a4968b0d38a4ccb8bc462, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/8f0b82ac584048f8a066bc031ec3dfab, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/586c4296929f42f38ce6178c20adb641, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/bbbc3416a9f94ef788e9beff130ff1ea, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/0766e4b1e3954c048cf194768da5d135, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/ba926f3391484103bdaa5af0bda39bc3, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/d1409acfca2741a7b2cd1c87f8e75fe1, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/10778409e00a4799a2c99c00a0eb3b8b, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/9b201c52a1b3495e9e6855cdf2d626eb, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/f5519ad431ce49d9843ef756ee056435, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/733c378e5c0946bd91d2c23618ff5e98, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/549a17f8fb734590bd54ae7cbc63f3ab, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/bbeca6763bc7417e8785b3674f69b8f2, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/0fbff8ee7a644b49b4eb7e600163170d, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/7946fc3f70e548128e52e71f83751aad, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/e05224a12920486b98eec06c4e53f1dc, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/3728bbc0a4084d99b757420f0bc82c9f, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/ce01e64c9d124a94a87d66fda7728a0d] to archive 2024-11-17T08:16:11,053 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T08:16:11,054 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/b7545be14c1a432f83eeb367c7f7eb9c to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/b7545be14c1a432f83eeb367c7f7eb9c 2024-11-17T08:16:11,055 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/ffb3418ecacf41ceae841c4e77d1e06f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/ffb3418ecacf41ceae841c4e77d1e06f 2024-11-17T08:16:11,056 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/0ba8e8a75621419fa50eb91f5da4edd7 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/0ba8e8a75621419fa50eb91f5da4edd7 2024-11-17T08:16:11,057 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/86132cf5fbc949b1b31513a745152274 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/86132cf5fbc949b1b31513a745152274 2024-11-17T08:16:11,058 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/f0ce9c39152a4968b0d38a4ccb8bc462 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/f0ce9c39152a4968b0d38a4ccb8bc462 2024-11-17T08:16:11,059 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/8f0b82ac584048f8a066bc031ec3dfab to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/8f0b82ac584048f8a066bc031ec3dfab 2024-11-17T08:16:11,060 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/586c4296929f42f38ce6178c20adb641 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/586c4296929f42f38ce6178c20adb641 2024-11-17T08:16:11,061 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/bbbc3416a9f94ef788e9beff130ff1ea to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/bbbc3416a9f94ef788e9beff130ff1ea 2024-11-17T08:16:11,062 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/0766e4b1e3954c048cf194768da5d135 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/0766e4b1e3954c048cf194768da5d135 2024-11-17T08:16:11,063 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/ba926f3391484103bdaa5af0bda39bc3 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/ba926f3391484103bdaa5af0bda39bc3 2024-11-17T08:16:11,063 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/d1409acfca2741a7b2cd1c87f8e75fe1 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/d1409acfca2741a7b2cd1c87f8e75fe1 2024-11-17T08:16:11,064 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/10778409e00a4799a2c99c00a0eb3b8b to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/10778409e00a4799a2c99c00a0eb3b8b 2024-11-17T08:16:11,065 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/9b201c52a1b3495e9e6855cdf2d626eb to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/9b201c52a1b3495e9e6855cdf2d626eb 2024-11-17T08:16:11,065 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/f5519ad431ce49d9843ef756ee056435 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/f5519ad431ce49d9843ef756ee056435 2024-11-17T08:16:11,066 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/733c378e5c0946bd91d2c23618ff5e98 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/733c378e5c0946bd91d2c23618ff5e98 2024-11-17T08:16:11,067 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/549a17f8fb734590bd54ae7cbc63f3ab to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/549a17f8fb734590bd54ae7cbc63f3ab 2024-11-17T08:16:11,067 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/bbeca6763bc7417e8785b3674f69b8f2 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/bbeca6763bc7417e8785b3674f69b8f2 2024-11-17T08:16:11,068 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/0fbff8ee7a644b49b4eb7e600163170d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/0fbff8ee7a644b49b4eb7e600163170d 2024-11-17T08:16:11,069 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/7946fc3f70e548128e52e71f83751aad to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/7946fc3f70e548128e52e71f83751aad 2024-11-17T08:16:11,070 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/e05224a12920486b98eec06c4e53f1dc to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/e05224a12920486b98eec06c4e53f1dc 2024-11-17T08:16:11,071 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/3728bbc0a4084d99b757420f0bc82c9f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/3728bbc0a4084d99b757420f0bc82c9f 2024-11-17T08:16:11,072 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/ce01e64c9d124a94a87d66fda7728a0d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/ce01e64c9d124a94a87d66fda7728a0d 2024-11-17T08:16:11,076 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/d37e6908cb3f4f0a96e3608dca56ed45, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/0e0afce9248c4c679dabb0d639d5bd7c, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/ceeae6fb0a524d978b5e762a5ad33a4a, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/fb195d63e9754afaad855395aa5ad959, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/4a3c8d88746e4e5682ed23bede52e0b8, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/6b058b0db96d4faa92357c9cc855cf69, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/12b9af7ad4a24faba40517eb010b06a8, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/b22077599bcd450d987ec1b24324f3e6, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/a6381decb42341bea2ce07f69368babf, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/4735c6dc7f5a47ecbd9397f14a519513, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/9cf4d97e5beb49629fd5bdacb34ce11a, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/d8ddea08cc604e0fb91311f82aad4ceb, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/259cc73715da44279dc50522a99ce362, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/e8e89fd14b534b168ed4dfb3f0908d4b, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/f3550537109449469a45ec194c42e48e, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/68ce81ee75c849b59e78362a2e79d8cd, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/617a8b1d87da44f79bc2972745708b2b, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/9890ff7173a14b548da6ef4172bd003d, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/14de48c9035c4ca6aa52e68ed532a508, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/39927293fcbf4461b42859f6940443fd, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/d327c6d8f5d14251bae62de9b53b159d, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/86a2452142524f60a25f6e01d4b4b507] to archive 2024-11-17T08:16:11,077 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T08:16:11,078 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/d37e6908cb3f4f0a96e3608dca56ed45 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/d37e6908cb3f4f0a96e3608dca56ed45 2024-11-17T08:16:11,080 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/0e0afce9248c4c679dabb0d639d5bd7c to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/0e0afce9248c4c679dabb0d639d5bd7c 2024-11-17T08:16:11,081 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/ceeae6fb0a524d978b5e762a5ad33a4a to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/ceeae6fb0a524d978b5e762a5ad33a4a 2024-11-17T08:16:11,090 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/fb195d63e9754afaad855395aa5ad959 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/fb195d63e9754afaad855395aa5ad959 2024-11-17T08:16:11,094 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/4a3c8d88746e4e5682ed23bede52e0b8 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/4a3c8d88746e4e5682ed23bede52e0b8 2024-11-17T08:16:11,096 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/6b058b0db96d4faa92357c9cc855cf69 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/6b058b0db96d4faa92357c9cc855cf69 2024-11-17T08:16:11,100 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/12b9af7ad4a24faba40517eb010b06a8 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/12b9af7ad4a24faba40517eb010b06a8 2024-11-17T08:16:11,102 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/b22077599bcd450d987ec1b24324f3e6 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/b22077599bcd450d987ec1b24324f3e6 2024-11-17T08:16:11,103 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/a6381decb42341bea2ce07f69368babf to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/a6381decb42341bea2ce07f69368babf 2024-11-17T08:16:11,107 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/4735c6dc7f5a47ecbd9397f14a519513 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/4735c6dc7f5a47ecbd9397f14a519513 2024-11-17T08:16:11,108 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/9cf4d97e5beb49629fd5bdacb34ce11a to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/9cf4d97e5beb49629fd5bdacb34ce11a 2024-11-17T08:16:11,109 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/d8ddea08cc604e0fb91311f82aad4ceb to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/d8ddea08cc604e0fb91311f82aad4ceb 2024-11-17T08:16:11,110 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/259cc73715da44279dc50522a99ce362 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/259cc73715da44279dc50522a99ce362 2024-11-17T08:16:11,112 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/e8e89fd14b534b168ed4dfb3f0908d4b to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/e8e89fd14b534b168ed4dfb3f0908d4b 2024-11-17T08:16:11,113 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/f3550537109449469a45ec194c42e48e to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/f3550537109449469a45ec194c42e48e 2024-11-17T08:16:11,114 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/68ce81ee75c849b59e78362a2e79d8cd to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/68ce81ee75c849b59e78362a2e79d8cd 2024-11-17T08:16:11,115 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/617a8b1d87da44f79bc2972745708b2b to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/617a8b1d87da44f79bc2972745708b2b 2024-11-17T08:16:11,116 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/9890ff7173a14b548da6ef4172bd003d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/9890ff7173a14b548da6ef4172bd003d 2024-11-17T08:16:11,117 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/14de48c9035c4ca6aa52e68ed532a508 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/14de48c9035c4ca6aa52e68ed532a508 2024-11-17T08:16:11,118 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/39927293fcbf4461b42859f6940443fd to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/39927293fcbf4461b42859f6940443fd 2024-11-17T08:16:11,120 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/d327c6d8f5d14251bae62de9b53b159d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/d327c6d8f5d14251bae62de9b53b159d 2024-11-17T08:16:11,124 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/86a2452142524f60a25f6e01d4b4b507 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/86a2452142524f60a25f6e01d4b4b507 2024-11-17T08:16:11,129 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/9a33c619f6a44d20bdf8a75cff1338a5, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/d4078f8657454a7baf481a8fcf606cd9, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/2341bb1c27574b838b5c811ebca77caa, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/5d81b47fdb964653a00df56c6a12395f, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/2d03be7081de4ee3a282d7070b4a3938, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/eadbac69351f48429df54b2d22b21fd9, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/6e51d6dbe51c4b55bdb7136e3bc7ed73, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/248732c3099d49aca3f167641c1779e7, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/a8ba04f8e6fd45b19cae086707ec3c1d, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/e1882648e79e4860b40eaac828d576f8, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/ddbd2a788abd49bfabf7beb82de1793e, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/20ad3be8a0e14e7cb3639a9fdde3c762, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/289da5810f6346a3939eead8866dde14, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/820d985e048d4130b3f3507147c1861d, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/db11d2e71b6e476687e6d0d2aaae25c5, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/c0384260db4f42469aedafb0b2d74ee0, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/21a5f1ce5e764e6591fa809f96ba90a8, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/b6d72549b3fc4c9582bb285b26e4847a, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/b66925eea51946d1b84b1950e629829b, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/9de13eb229ef4cdeab936b50c177be9c, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/bcef6d5ec73140a68e2866cfa494e835, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/1a0d82e32f7d405d865ced75ad3f778a] to archive 2024-11-17T08:16:11,131 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T08:16:11,133 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/9a33c619f6a44d20bdf8a75cff1338a5 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/9a33c619f6a44d20bdf8a75cff1338a5 2024-11-17T08:16:11,136 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/d4078f8657454a7baf481a8fcf606cd9 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/d4078f8657454a7baf481a8fcf606cd9 2024-11-17T08:16:11,137 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/2341bb1c27574b838b5c811ebca77caa to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/2341bb1c27574b838b5c811ebca77caa 2024-11-17T08:16:11,140 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/5d81b47fdb964653a00df56c6a12395f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/5d81b47fdb964653a00df56c6a12395f 2024-11-17T08:16:11,142 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/2d03be7081de4ee3a282d7070b4a3938 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/2d03be7081de4ee3a282d7070b4a3938 2024-11-17T08:16:11,143 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/eadbac69351f48429df54b2d22b21fd9 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/eadbac69351f48429df54b2d22b21fd9 2024-11-17T08:16:11,144 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/6e51d6dbe51c4b55bdb7136e3bc7ed73 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/6e51d6dbe51c4b55bdb7136e3bc7ed73 2024-11-17T08:16:11,145 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/248732c3099d49aca3f167641c1779e7 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/248732c3099d49aca3f167641c1779e7 2024-11-17T08:16:11,146 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/a8ba04f8e6fd45b19cae086707ec3c1d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/a8ba04f8e6fd45b19cae086707ec3c1d 2024-11-17T08:16:11,147 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/e1882648e79e4860b40eaac828d576f8 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/e1882648e79e4860b40eaac828d576f8 2024-11-17T08:16:11,148 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/ddbd2a788abd49bfabf7beb82de1793e to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/ddbd2a788abd49bfabf7beb82de1793e 2024-11-17T08:16:11,149 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/20ad3be8a0e14e7cb3639a9fdde3c762 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/20ad3be8a0e14e7cb3639a9fdde3c762 2024-11-17T08:16:11,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T08:16:11,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:11,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:16:11,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:11,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:11,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:11,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:11,152 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/289da5810f6346a3939eead8866dde14 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/289da5810f6346a3939eead8866dde14 2024-11-17T08:16:11,153 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/820d985e048d4130b3f3507147c1861d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/820d985e048d4130b3f3507147c1861d 2024-11-17T08:16:11,154 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/db11d2e71b6e476687e6d0d2aaae25c5 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/db11d2e71b6e476687e6d0d2aaae25c5 2024-11-17T08:16:11,155 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/c0384260db4f42469aedafb0b2d74ee0 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/c0384260db4f42469aedafb0b2d74ee0 2024-11-17T08:16:11,157 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/21a5f1ce5e764e6591fa809f96ba90a8 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/21a5f1ce5e764e6591fa809f96ba90a8 2024-11-17T08:16:11,158 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/b6d72549b3fc4c9582bb285b26e4847a to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/b6d72549b3fc4c9582bb285b26e4847a 2024-11-17T08:16:11,159 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/b66925eea51946d1b84b1950e629829b to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/b66925eea51946d1b84b1950e629829b 2024-11-17T08:16:11,160 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/9de13eb229ef4cdeab936b50c177be9c to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/9de13eb229ef4cdeab936b50c177be9c 2024-11-17T08:16:11,161 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/bcef6d5ec73140a68e2866cfa494e835 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/bcef6d5ec73140a68e2866cfa494e835 2024-11-17T08:16:11,164 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/1a0d82e32f7d405d865ced75ad3f778a to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/1a0d82e32f7d405d865ced75ad3f778a 2024-11-17T08:16:11,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:11,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831431203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:11,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:11,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831431203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:11,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:11,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:11,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:11,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:11,215 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:11,215 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:11,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831431203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:11,215 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:11,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:11,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:11,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:11,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:11,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831431203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:11,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:11,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:11,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:11,303 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=672, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/a427f1a4533e4ffba1cd0ccc6492ffdb 2024-11-17T08:16:11,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T08:16:11,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:11,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:16:11,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:11,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:11,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:11,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:11,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/7b8af905ba5c4ee29184f0bf8bb99148 is 50, key is test_row_0/B:col10/1731831370476/Put/seqid=0 2024-11-17T08:16:11,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742250_1426 (size=12301) 2024-11-17T08:16:11,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T08:16:11,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:11,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:16:11,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:11,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:11,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:11,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:11,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T08:16:11,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:11,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:16:11,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:11,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:11,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:11,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:11,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=672 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/7b8af905ba5c4ee29184f0bf8bb99148 2024-11-17T08:16:11,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/d96b3b556f624d5d8055e1f31cd87746 is 50, key is test_row_0/C:col10/1731831370476/Put/seqid=0 2024-11-17T08:16:11,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742251_1427 (size=12301) 2024-11-17T08:16:11,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:11,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53472 deadline: 1731831431723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:11,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:11,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53546 deadline: 1731831431723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:11,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:11,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:11,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:11,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:11,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:11,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:11,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1731831431723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:11,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:11,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:11,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:11,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:11,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:11,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53500 deadline: 1731831431723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:11,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:11,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:11,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:11,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T08:16:11,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:11,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:16:11,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:11,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:11,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:11,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:11,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T08:16:11,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:11,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:16:11,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:11,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:11,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:11,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:12,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T08:16:12,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:12,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:16:12,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:12,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:12,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:12,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:12,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=672 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/d96b3b556f624d5d8055e1f31cd87746 2024-11-17T08:16:12,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/a427f1a4533e4ffba1cd0ccc6492ffdb as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/a427f1a4533e4ffba1cd0ccc6492ffdb 2024-11-17T08:16:12,136 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/a427f1a4533e4ffba1cd0ccc6492ffdb, entries=200, sequenceid=672, filesize=39.0 K 2024-11-17T08:16:12,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/7b8af905ba5c4ee29184f0bf8bb99148 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/7b8af905ba5c4ee29184f0bf8bb99148 2024-11-17T08:16:12,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/7b8af905ba5c4ee29184f0bf8bb99148, entries=150, sequenceid=672, filesize=12.0 K 2024-11-17T08:16:12,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/d96b3b556f624d5d8055e1f31cd87746 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/d96b3b556f624d5d8055e1f31cd87746 2024-11-17T08:16:12,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/d96b3b556f624d5d8055e1f31cd87746, entries=150, sequenceid=672, filesize=12.0 K 2024-11-17T08:16:12,144 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~185.81 KB/190272, currentSize=114.05 KB/116790 for b852432b87998249586edaadc388ac84 in 1660ms, sequenceid=672, compaction requested=true 2024-11-17T08:16:12,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b852432b87998249586edaadc388ac84: 2024-11-17T08:16:12,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b852432b87998249586edaadc388ac84:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:16:12,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:12,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b852432b87998249586edaadc388ac84:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:16:12,144 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:16:12,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:12,144 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:16:12,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b852432b87998249586edaadc388ac84:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:16:12,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:16:12,145 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111903 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:16:12,145 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:16:12,145 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): b852432b87998249586edaadc388ac84/A is initiating minor compaction (all files) 2024-11-17T08:16:12,145 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): b852432b87998249586edaadc388ac84/B is initiating minor compaction (all files) 2024-11-17T08:16:12,145 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b852432b87998249586edaadc388ac84/B in TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:12,145 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b852432b87998249586edaadc388ac84/A in TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:12,145 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/fc41cefac1fb43e680e11ed755e379f3, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/157be605c56848609887dd0ae3894145, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/a427f1a4533e4ffba1cd0ccc6492ffdb] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp, totalSize=109.3 K 2024-11-17T08:16:12,145 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/94956228aab84fafbcc13c5ebfdbd8ef, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/0f77caab5cef4ef2b8133e446eba44ae, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/7b8af905ba5c4ee29184f0bf8bb99148] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp, totalSize=36.8 K 2024-11-17T08:16:12,145 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:12,145 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. files: [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/fc41cefac1fb43e680e11ed755e379f3, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/157be605c56848609887dd0ae3894145, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/a427f1a4533e4ffba1cd0ccc6492ffdb] 2024-11-17T08:16:12,145 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 94956228aab84fafbcc13c5ebfdbd8ef, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731831368261 2024-11-17T08:16:12,145 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc41cefac1fb43e680e11ed755e379f3, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731831368261 2024-11-17T08:16:12,145 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f77caab5cef4ef2b8133e446eba44ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731831368397 2024-11-17T08:16:12,146 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b8af905ba5c4ee29184f0bf8bb99148, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=672, earliestPutTs=1731831370424 2024-11-17T08:16:12,146 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 157be605c56848609887dd0ae3894145, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731831368397 2024-11-17T08:16:12,146 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting a427f1a4533e4ffba1cd0ccc6492ffdb, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=672, earliestPutTs=1731831370424 2024-11-17T08:16:12,150 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:16:12,152 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b852432b87998249586edaadc388ac84#B#compaction#357 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:12,152 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/57a3c12bc4ba400ba3eb00d0dd2ef283 is 50, key is test_row_0/B:col10/1731831370476/Put/seqid=0 2024-11-17T08:16:12,154 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111747dea3a47a934c55a605bf8bf2f57899_b852432b87998249586edaadc388ac84 store=[table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:16:12,155 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111747dea3a47a934c55a605bf8bf2f57899_b852432b87998249586edaadc388ac84, store=[table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:16:12,156 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111747dea3a47a934c55a605bf8bf2f57899_b852432b87998249586edaadc388ac84 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:16:12,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742252_1428 (size=12404) 2024-11-17T08:16:12,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742253_1429 (size=4469) 2024-11-17T08:16:12,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T08:16:12,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:12,221 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing b852432b87998249586edaadc388ac84 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-17T08:16:12,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=A 2024-11-17T08:16:12,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:12,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=B 2024-11-17T08:16:12,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:12,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=C 2024-11-17T08:16:12,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:12,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111794baed8507d94869bc51c97ccd045ec6_b852432b87998249586edaadc388ac84 is 50, key is test_row_0/A:col10/1731831370546/Put/seqid=0 2024-11-17T08:16:12,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742254_1430 (size=12454) 2024-11-17T08:16:12,571 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/57a3c12bc4ba400ba3eb00d0dd2ef283 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/57a3c12bc4ba400ba3eb00d0dd2ef283 2024-11-17T08:16:12,574 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b852432b87998249586edaadc388ac84#A#compaction#358 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:12,574 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b852432b87998249586edaadc388ac84/B of b852432b87998249586edaadc388ac84 into 57a3c12bc4ba400ba3eb00d0dd2ef283(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:12,574 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b852432b87998249586edaadc388ac84: 2024-11-17T08:16:12,574 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., storeName=b852432b87998249586edaadc388ac84/B, priority=13, startTime=1731831372144; duration=0sec 2024-11-17T08:16:12,574 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/83052bf5c0da4d029afe93da4c2c43ca is 175, key is test_row_0/A:col10/1731831370476/Put/seqid=0 2024-11-17T08:16:12,574 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:16:12,574 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b852432b87998249586edaadc388ac84:B 2024-11-17T08:16:12,575 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:16:12,575 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:16:12,575 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): b852432b87998249586edaadc388ac84/C is initiating minor compaction (all files) 2024-11-17T08:16:12,575 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b852432b87998249586edaadc388ac84/C in TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:12,575 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/0eb0c229b5d04b27aaeb18fa08eba802, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/a06f25ba877f42739f4f0c62e0e3d89c, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/d96b3b556f624d5d8055e1f31cd87746] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp, totalSize=36.8 K 2024-11-17T08:16:12,576 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 0eb0c229b5d04b27aaeb18fa08eba802, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731831368261 2024-11-17T08:16:12,576 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting a06f25ba877f42739f4f0c62e0e3d89c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731831368397 2024-11-17T08:16:12,576 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting d96b3b556f624d5d8055e1f31cd87746, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=672, earliestPutTs=1731831370424 2024-11-17T08:16:12,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742255_1431 (size=31358) 2024-11-17T08:16:12,583 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b852432b87998249586edaadc388ac84#C#compaction#360 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:12,583 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/d0daf60322534860b3e94fd40a0255be is 50, key is test_row_0/C:col10/1731831370476/Put/seqid=0 2024-11-17T08:16:12,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742256_1432 (size=12404) 2024-11-17T08:16:12,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on b852432b87998249586edaadc388ac84 2024-11-17T08:16:12,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. as already flushing 2024-11-17T08:16:12,611 INFO [Thread-1501 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1501. 2024-11-17T08:16:12,611 DEBUG [Thread-1501 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:16:12,611 DEBUG [Thread-1501 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:12,611 DEBUG [Thread-1501 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:12,611 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:16:12,612 INFO [Thread-1516 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1516. 2024-11-17T08:16:12,612 DEBUG [Thread-1516 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:16:12,612 INFO [Thread-1514 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1514. 2024-11-17T08:16:12,612 DEBUG [Thread-1516 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:12,612 DEBUG [Thread-1514 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:16:12,612 DEBUG [Thread-1514 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:12,612 DEBUG [Thread-1516 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:12,612 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:16:12,612 DEBUG [Thread-1514 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:12,612 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:16:12,614 INFO [Thread-1510 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1510. 2024-11-17T08:16:12,614 DEBUG [Thread-1510 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:16:12,614 DEBUG [Thread-1510 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:12,614 DEBUG [Thread-1510 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:12,615 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:16:12,620 INFO [Thread-1518 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1518. 2024-11-17T08:16:12,620 DEBUG [Thread-1518 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:16:12,620 DEBUG [Thread-1518 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:12,620 DEBUG [Thread-1518 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:12,620 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:16:12,620 INFO [Thread-1512 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1512. 2024-11-17T08:16:12,620 DEBUG [Thread-1512 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:16:12,620 DEBUG [Thread-1512 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:12,621 DEBUG [Thread-1512 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:12,621 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:16:12,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:12,637 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111794baed8507d94869bc51c97ccd045ec6_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111794baed8507d94869bc51c97ccd045ec6_b852432b87998249586edaadc388ac84 2024-11-17T08:16:12,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/336fd594a3db4fd0b30e4de9b636a295, store: [table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:16:12,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/336fd594a3db4fd0b30e4de9b636a295 is 175, key is test_row_0/A:col10/1731831370546/Put/seqid=0 2024-11-17T08:16:12,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742257_1433 (size=31255) 2024-11-17T08:16:12,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T08:16:12,749 INFO [Thread-1507 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1507. 2024-11-17T08:16:12,749 INFO [Thread-1503 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1503. 2024-11-17T08:16:12,749 DEBUG [Thread-1507 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:16:12,749 DEBUG [Thread-1503 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:16:12,749 DEBUG [Thread-1507 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:12,749 DEBUG [Thread-1503 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:12,750 DEBUG [Thread-1507 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:12,750 DEBUG [Thread-1503 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:12,750 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:16:12,750 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:16:12,750 INFO [Thread-1505 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1505. 2024-11-17T08:16:12,750 DEBUG [Thread-1505 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:16:12,751 DEBUG [Thread-1505 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:12,751 DEBUG [Thread-1505 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:12,751 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:16:12,752 INFO [Thread-1499 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1499. 2024-11-17T08:16:12,753 DEBUG [Thread-1499 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:16:12,753 DEBUG [Thread-1499 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:12,753 DEBUG [Thread-1499 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:12,753 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:16:12,982 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/83052bf5c0da4d029afe93da4c2c43ca as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/83052bf5c0da4d029afe93da4c2c43ca 2024-11-17T08:16:12,987 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b852432b87998249586edaadc388ac84/A of b852432b87998249586edaadc388ac84 into 83052bf5c0da4d029afe93da4c2c43ca(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:12,987 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b852432b87998249586edaadc388ac84: 2024-11-17T08:16:12,987 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., storeName=b852432b87998249586edaadc388ac84/A, priority=13, startTime=1731831372144; duration=0sec 2024-11-17T08:16:12,987 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:12,987 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b852432b87998249586edaadc388ac84:A 2024-11-17T08:16:12,991 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/d0daf60322534860b3e94fd40a0255be as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/d0daf60322534860b3e94fd40a0255be 2024-11-17T08:16:12,996 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b852432b87998249586edaadc388ac84/C of b852432b87998249586edaadc388ac84 into d0daf60322534860b3e94fd40a0255be(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:12,997 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b852432b87998249586edaadc388ac84: 2024-11-17T08:16:12,997 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., storeName=b852432b87998249586edaadc388ac84/C, priority=13, startTime=1731831372144; duration=0sec 2024-11-17T08:16:12,997 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:12,997 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b852432b87998249586edaadc388ac84:C 2024-11-17T08:16:13,042 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=692, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/336fd594a3db4fd0b30e4de9b636a295 2024-11-17T08:16:13,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/9ebf0dd376e24e6ea4982ab4b5cd0c0a is 50, key is test_row_0/B:col10/1731831370546/Put/seqid=0 2024-11-17T08:16:13,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742258_1434 (size=12301) 2024-11-17T08:16:13,451 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=692 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/9ebf0dd376e24e6ea4982ab4b5cd0c0a 2024-11-17T08:16:13,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/853626ec22ca4d41bd796552d496dbb3 is 50, key is test_row_0/C:col10/1731831370546/Put/seqid=0 2024-11-17T08:16:13,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742259_1435 (size=12301) 2024-11-17T08:16:13,863 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=692 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/853626ec22ca4d41bd796552d496dbb3 2024-11-17T08:16:13,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/336fd594a3db4fd0b30e4de9b636a295 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/336fd594a3db4fd0b30e4de9b636a295 2024-11-17T08:16:13,877 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/336fd594a3db4fd0b30e4de9b636a295, entries=150, sequenceid=692, filesize=30.5 K 2024-11-17T08:16:13,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/9ebf0dd376e24e6ea4982ab4b5cd0c0a as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/9ebf0dd376e24e6ea4982ab4b5cd0c0a 2024-11-17T08:16:13,882 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/9ebf0dd376e24e6ea4982ab4b5cd0c0a, entries=150, sequenceid=692, filesize=12.0 K 2024-11-17T08:16:13,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/853626ec22ca4d41bd796552d496dbb3 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/853626ec22ca4d41bd796552d496dbb3 2024-11-17T08:16:13,886 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/853626ec22ca4d41bd796552d496dbb3, entries=150, sequenceid=692, filesize=12.0 K 2024-11-17T08:16:13,887 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=40.25 KB/41220 for b852432b87998249586edaadc388ac84 in 1665ms, sequenceid=692, compaction requested=false 2024-11-17T08:16:13,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for b852432b87998249586edaadc388ac84: 2024-11-17T08:16:13,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:13,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-17T08:16:13,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-17T08:16:13,890 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-17T08:16:13,890 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 5.3420 sec 2024-11-17T08:16:13,892 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 5.3470 sec 2024-11-17T08:16:16,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T08:16:16,725 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:16:16,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T08:16:16,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 162 2024-11-17T08:16:16,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-11-17T08:16:16,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 153 2024-11-17T08:16:16,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-17T08:16:16,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 147 2024-11-17T08:16:16,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T08:16:16,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T08:16:16,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 939 2024-11-17T08:16:16,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2811 rows 2024-11-17T08:16:16,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 941 2024-11-17T08:16:16,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2822 rows 2024-11-17T08:16:16,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 929 2024-11-17T08:16:16,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2783 rows 2024-11-17T08:16:16,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 941 2024-11-17T08:16:16,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2818 rows 2024-11-17T08:16:16,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 936 2024-11-17T08:16:16,725 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2798 rows 2024-11-17T08:16:16,725 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T08:16:16,725 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T08:16:16,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:16,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:16,725 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:16:16,726 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-17T08:16:16,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T08:16:16,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-17T08:16:16,730 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731831376730"}]},"ts":"1731831376730"} 2024-11-17T08:16:16,732 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T08:16:16,732 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T08:16:16,733 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T08:16:16,736 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b852432b87998249586edaadc388ac84, UNASSIGN}] 2024-11-17T08:16:16,736 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b852432b87998249586edaadc388ac84, UNASSIGN 2024-11-17T08:16:16,737 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=b852432b87998249586edaadc388ac84, regionState=CLOSING, regionLocation=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:16,739 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b852432b87998249586edaadc388ac84, UNASSIGN because future has completed 2024-11-17T08:16:16,739 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T08:16:16,739 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805}] 2024-11-17T08:16:16,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-17T08:16:16,892 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(122): Close b852432b87998249586edaadc388ac84 2024-11-17T08:16:16,892 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T08:16:16,892 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1722): Closing b852432b87998249586edaadc388ac84, disabling compactions & flushes 2024-11-17T08:16:16,892 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:16,892 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:16,892 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. after waiting 0 ms 2024-11-17T08:16:16,892 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:16,892 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(2902): Flushing b852432b87998249586edaadc388ac84 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-17T08:16:16,892 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=A 2024-11-17T08:16:16,893 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:16,893 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=B 2024-11-17T08:16:16,893 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:16,893 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b852432b87998249586edaadc388ac84, store=C 2024-11-17T08:16:16,893 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:16,898 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d94a96afab56479e8ffb983344398f99_b852432b87998249586edaadc388ac84 is 50, key is test_row_0/A:col10/1731831372745/Put/seqid=0 2024-11-17T08:16:16,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742260_1436 (size=12454) 2024-11-17T08:16:17,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-17T08:16:17,302 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:17,311 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d94a96afab56479e8ffb983344398f99_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d94a96afab56479e8ffb983344398f99_b852432b87998249586edaadc388ac84 2024-11-17T08:16:17,313 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/97450ffc15ce43a299cbdda5f1f7bd65, store: [table=TestAcidGuarantees family=A region=b852432b87998249586edaadc388ac84] 2024-11-17T08:16:17,313 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/97450ffc15ce43a299cbdda5f1f7bd65 is 175, key is test_row_0/A:col10/1731831372745/Put/seqid=0 2024-11-17T08:16:17,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742261_1437 (size=31255) 2024-11-17T08:16:17,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-17T08:16:17,719 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=704, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/97450ffc15ce43a299cbdda5f1f7bd65 2024-11-17T08:16:17,729 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/5ebbd53a526741b4934318fc62c7a22b is 50, key is test_row_0/B:col10/1731831372745/Put/seqid=0 2024-11-17T08:16:17,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742262_1438 (size=12301) 2024-11-17T08:16:17,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-17T08:16:18,134 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=704 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/5ebbd53a526741b4934318fc62c7a22b 2024-11-17T08:16:18,151 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/114442150273477aa33d959d9121fc2f is 50, key is test_row_0/C:col10/1731831372745/Put/seqid=0 2024-11-17T08:16:18,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742263_1439 (size=12301) 2024-11-17T08:16:18,557 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=704 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/114442150273477aa33d959d9121fc2f 2024-11-17T08:16:18,566 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/A/97450ffc15ce43a299cbdda5f1f7bd65 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/97450ffc15ce43a299cbdda5f1f7bd65 2024-11-17T08:16:18,572 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/97450ffc15ce43a299cbdda5f1f7bd65, entries=150, sequenceid=704, filesize=30.5 K 2024-11-17T08:16:18,573 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/B/5ebbd53a526741b4934318fc62c7a22b as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/5ebbd53a526741b4934318fc62c7a22b 2024-11-17T08:16:18,578 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/5ebbd53a526741b4934318fc62c7a22b, entries=150, sequenceid=704, filesize=12.0 K 2024-11-17T08:16:18,579 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/.tmp/C/114442150273477aa33d959d9121fc2f as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/114442150273477aa33d959d9121fc2f 2024-11-17T08:16:18,583 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/114442150273477aa33d959d9121fc2f, entries=150, sequenceid=704, filesize=12.0 K 2024-11-17T08:16:18,584 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=0 B/0 for b852432b87998249586edaadc388ac84 in 1692ms, sequenceid=704, compaction requested=true 2024-11-17T08:16:18,585 DEBUG [StoreCloser-TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/fc41cefac1fb43e680e11ed755e379f3, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/157be605c56848609887dd0ae3894145, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/a427f1a4533e4ffba1cd0ccc6492ffdb] to archive 2024-11-17T08:16:18,585 DEBUG [StoreCloser-TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T08:16:18,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/fc41cefac1fb43e680e11ed755e379f3 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/fc41cefac1fb43e680e11ed755e379f3 2024-11-17T08:16:18,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/157be605c56848609887dd0ae3894145 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/157be605c56848609887dd0ae3894145 2024-11-17T08:16:18,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/a427f1a4533e4ffba1cd0ccc6492ffdb to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/a427f1a4533e4ffba1cd0ccc6492ffdb 2024-11-17T08:16:18,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/94956228aab84fafbcc13c5ebfdbd8ef, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/0f77caab5cef4ef2b8133e446eba44ae, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/7b8af905ba5c4ee29184f0bf8bb99148] to archive 2024-11-17T08:16:18,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T08:16:18,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/94956228aab84fafbcc13c5ebfdbd8ef to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/94956228aab84fafbcc13c5ebfdbd8ef 2024-11-17T08:16:18,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/0f77caab5cef4ef2b8133e446eba44ae to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/0f77caab5cef4ef2b8133e446eba44ae 2024-11-17T08:16:18,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/7b8af905ba5c4ee29184f0bf8bb99148 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/7b8af905ba5c4ee29184f0bf8bb99148 2024-11-17T08:16:18,594 DEBUG [StoreCloser-TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/0eb0c229b5d04b27aaeb18fa08eba802, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/a06f25ba877f42739f4f0c62e0e3d89c, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/d96b3b556f624d5d8055e1f31cd87746] to archive 2024-11-17T08:16:18,594 DEBUG [StoreCloser-TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T08:16:18,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/0eb0c229b5d04b27aaeb18fa08eba802 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/0eb0c229b5d04b27aaeb18fa08eba802 2024-11-17T08:16:18,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/a06f25ba877f42739f4f0c62e0e3d89c to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/a06f25ba877f42739f4f0c62e0e3d89c 2024-11-17T08:16:18,597 DEBUG [StoreCloser-TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/d96b3b556f624d5d8055e1f31cd87746 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/d96b3b556f624d5d8055e1f31cd87746 2024-11-17T08:16:18,600 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/recovered.edits/707.seqid, newMaxSeqId=707, maxSeqId=4 2024-11-17T08:16:18,601 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. 2024-11-17T08:16:18,601 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1676): Region close journal for b852432b87998249586edaadc388ac84: Waiting for close lock at 1731831376892Running coprocessor pre-close hooks at 1731831376892Disabling compacts and flushes for region at 1731831376892Disabling writes for close at 1731831376892Obtaining lock to block concurrent updates at 1731831376892Preparing flush snapshotting stores in b852432b87998249586edaadc388ac84 at 1731831376892Finished memstore snapshotting TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84., syncing WAL and waiting on mvcc, flushsize=dataSize=41220, getHeapSize=123120, getOffHeapSize=0, getCellsCount=900 at 1731831376893 (+1 ms)Flushing stores of TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84. at 1731831376893Flushing b852432b87998249586edaadc388ac84/A: creating writer at 1731831376893Flushing mob file b852432b87998249586edaadc388ac84/A: appending metadata at 1731831376898 (+5 ms)Flushing mob file b852432b87998249586edaadc388ac84/A: closing flushed file at 1731831376898Flushing b852432b87998249586edaadc388ac84/A: appending metadata at 1731831377313 (+415 ms)Flushing b852432b87998249586edaadc388ac84/A: closing flushed file at 1731831377313Flushing b852432b87998249586edaadc388ac84/B: creating writer at 1731831377726 (+413 ms)Flushing b852432b87998249586edaadc388ac84/B: appending metadata at 1731831377729 (+3 ms)Flushing b852432b87998249586edaadc388ac84/B: closing flushed file at 1731831377729Flushing b852432b87998249586edaadc388ac84/C: creating writer at 1731831378144 (+415 ms)Flushing b852432b87998249586edaadc388ac84/C: appending metadata at 1731831378150 (+6 ms)Flushing b852432b87998249586edaadc388ac84/C: closing flushed file at 1731831378150Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39f9312a: reopening flushed file at 1731831378564 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2961b5de: reopening flushed file at 1731831378572 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21555b9f: reopening flushed file at 1731831378578 (+6 ms)Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=0 B/0 for b852432b87998249586edaadc388ac84 in 1692ms, sequenceid=704, compaction requested=true at 1731831378584 (+6 ms)Writing region close event to WAL at 1731831378598 (+14 ms)Running coprocessor post-close hooks at 1731831378600 (+2 ms)Closed at 1731831378601 (+1 ms) 2024-11-17T08:16:18,602 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(157): Closed b852432b87998249586edaadc388ac84 2024-11-17T08:16:18,603 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=b852432b87998249586edaadc388ac84, regionState=CLOSED 2024-11-17T08:16:18,604 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 because future has completed 2024-11-17T08:16:18,606 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-17T08:16:18,606 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; CloseRegionProcedure b852432b87998249586edaadc388ac84, server=e3d0adb87c01,40631,1731831247805 in 1.8650 sec 2024-11-17T08:16:18,607 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-11-17T08:16:18,607 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b852432b87998249586edaadc388ac84, UNASSIGN in 1.8710 sec 2024-11-17T08:16:18,609 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-17T08:16:18,609 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8740 sec 2024-11-17T08:16:18,610 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731831378610"}]},"ts":"1731831378610"} 2024-11-17T08:16:18,611 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T08:16:18,611 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T08:16:18,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8860 sec 2024-11-17T08:16:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-17T08:16:18,876 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:16:18,877 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-17T08:16:18,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T08:16:18,882 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T08:16:18,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-17T08:16:18,883 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=119, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T08:16:18,886 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84 2024-11-17T08:16:18,888 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A, FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B, FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C, FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/recovered.edits] 2024-11-17T08:16:18,890 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/336fd594a3db4fd0b30e4de9b636a295 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/336fd594a3db4fd0b30e4de9b636a295 2024-11-17T08:16:18,891 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/83052bf5c0da4d029afe93da4c2c43ca to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/83052bf5c0da4d029afe93da4c2c43ca 2024-11-17T08:16:18,892 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/97450ffc15ce43a299cbdda5f1f7bd65 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/A/97450ffc15ce43a299cbdda5f1f7bd65 2024-11-17T08:16:18,893 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/57a3c12bc4ba400ba3eb00d0dd2ef283 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/57a3c12bc4ba400ba3eb00d0dd2ef283 2024-11-17T08:16:18,894 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/5ebbd53a526741b4934318fc62c7a22b to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/5ebbd53a526741b4934318fc62c7a22b 2024-11-17T08:16:18,894 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/9ebf0dd376e24e6ea4982ab4b5cd0c0a to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/B/9ebf0dd376e24e6ea4982ab4b5cd0c0a 2024-11-17T08:16:18,896 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/114442150273477aa33d959d9121fc2f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/114442150273477aa33d959d9121fc2f 2024-11-17T08:16:18,896 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/853626ec22ca4d41bd796552d496dbb3 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/853626ec22ca4d41bd796552d496dbb3 2024-11-17T08:16:18,897 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/d0daf60322534860b3e94fd40a0255be to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/C/d0daf60322534860b3e94fd40a0255be 2024-11-17T08:16:18,898 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/recovered.edits/707.seqid to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84/recovered.edits/707.seqid 2024-11-17T08:16:18,899 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/b852432b87998249586edaadc388ac84 2024-11-17T08:16:18,899 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T08:16:18,899 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T08:16:18,900 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-17T08:16:18,901 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117047572663e024b6589a220b21a695094_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117047572663e024b6589a220b21a695094_b852432b87998249586edaadc388ac84 2024-11-17T08:16:18,902 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411171bb0f4c8f35e49169e03f54343c71ef6_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411171bb0f4c8f35e49169e03f54343c71ef6_b852432b87998249586edaadc388ac84 2024-11-17T08:16:18,903 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111750f61e66d4884e6cbf5a3da1bcb50511_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111750f61e66d4884e6cbf5a3da1bcb50511_b852432b87998249586edaadc388ac84 2024-11-17T08:16:18,904 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175c4cf5508fb94846a3ee13fc6d7c763c_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175c4cf5508fb94846a3ee13fc6d7c763c_b852432b87998249586edaadc388ac84 2024-11-17T08:16:18,904 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176ec529d7aeab46db9846b5f729e79f4e_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176ec529d7aeab46db9846b5f729e79f4e_b852432b87998249586edaadc388ac84 2024-11-17T08:16:18,905 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176f0361e12a9b409e914d6e9450fc28fb_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176f0361e12a9b409e914d6e9450fc28fb_b852432b87998249586edaadc388ac84 2024-11-17T08:16:18,906 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176f898a54b0764fddb494d630d339d30a_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176f898a54b0764fddb494d630d339d30a_b852432b87998249586edaadc388ac84 2024-11-17T08:16:18,906 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111789df79b307124da18285257665d4ab0c_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111789df79b307124da18285257665d4ab0c_b852432b87998249586edaadc388ac84 2024-11-17T08:16:18,907 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178efd394900de40809710f23f8764c1c0_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178efd394900de40809710f23f8764c1c0_b852432b87998249586edaadc388ac84 2024-11-17T08:16:18,907 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117931077cf99a346fb9ee15872c71c7d78_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117931077cf99a346fb9ee15872c71c7d78_b852432b87998249586edaadc388ac84 2024-11-17T08:16:18,908 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111794baed8507d94869bc51c97ccd045ec6_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111794baed8507d94869bc51c97ccd045ec6_b852432b87998249586edaadc388ac84 2024-11-17T08:16:18,909 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411179ee3f56dc01449fdbbdb0aa33a2d26f4_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411179ee3f56dc01449fdbbdb0aa33a2d26f4_b852432b87998249586edaadc388ac84 2024-11-17T08:16:18,909 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b73ceb5284c2479e97b1e503a546938d_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b73ceb5284c2479e97b1e503a546938d_b852432b87998249586edaadc388ac84 2024-11-17T08:16:18,910 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b7b03d1bf7b14b098c8a4066a6e6fd44_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117b7b03d1bf7b14b098c8a4066a6e6fd44_b852432b87998249586edaadc388ac84 2024-11-17T08:16:18,911 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117cf24958beb4c42fc8d329318a2c65ea1_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117cf24958beb4c42fc8d329318a2c65ea1_b852432b87998249586edaadc388ac84 2024-11-17T08:16:18,911 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d112c3dd1c6c47bf82bff3e6febf1f35_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d112c3dd1c6c47bf82bff3e6febf1f35_b852432b87998249586edaadc388ac84 2024-11-17T08:16:18,912 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d89eabba5dea4b5d82f1c3a8dbd9ea5c_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d89eabba5dea4b5d82f1c3a8dbd9ea5c_b852432b87998249586edaadc388ac84 2024-11-17T08:16:18,913 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d94a96afab56479e8ffb983344398f99_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d94a96afab56479e8ffb983344398f99_b852432b87998249586edaadc388ac84 2024-11-17T08:16:18,913 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117eb1e0c3031ca4e509d0be5d8700dbce5_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117eb1e0c3031ca4e509d0be5d8700dbce5_b852432b87998249586edaadc388ac84 2024-11-17T08:16:18,914 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ec01eab81ada451692b7bdc32ac2195f_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ec01eab81ada451692b7bdc32ac2195f_b852432b87998249586edaadc388ac84 2024-11-17T08:16:18,915 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117edc8f131fd99499189a48485b9032bb4_b852432b87998249586edaadc388ac84 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117edc8f131fd99499189a48485b9032bb4_b852432b87998249586edaadc388ac84 2024-11-17T08:16:18,915 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T08:16:18,917 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=119, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T08:16:18,918 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T08:16:18,920 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T08:16:18,921 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=119, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T08:16:18,921 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T08:16:18,921 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731831378921"}]},"ts":"9223372036854775807"} 2024-11-17T08:16:18,922 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T08:16:18,922 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b852432b87998249586edaadc388ac84, NAME => 'TestAcidGuarantees,,1731831349471.b852432b87998249586edaadc388ac84.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T08:16:18,922 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T08:16:18,923 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731831378922"}]},"ts":"9223372036854775807"} 2024-11-17T08:16:18,924 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T08:16:18,924 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=119, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T08:16:18,925 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 47 msec 2024-11-17T08:16:18,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-17T08:16:18,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T08:16:18,995 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:16:19,003 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250 (was 249) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-306999293_22 at /127.0.0.1:37884 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/e3d0adb87c01:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=366 (was 312) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2060 (was 2410) 2024-11-17T08:16:19,011 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=366, ProcessCount=11, AvailableMemoryMB=2059 2024-11-17T08:16:19,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T08:16:19,012 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T08:16:19,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T08:16:19,014 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T08:16:19,014 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:19,014 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 120 2024-11-17T08:16:19,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-17T08:16:19,015 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T08:16:19,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742264_1440 (size=1061) 2024-11-17T08:16:19,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-17T08:16:19,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-17T08:16:19,425 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85 2024-11-17T08:16:19,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742265_1441 (size=53) 2024-11-17T08:16:19,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-17T08:16:19,832 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T08:16:19,832 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c34c3478acc81ba3f3c8f5caec36c13b, disabling compactions & flushes 2024-11-17T08:16:19,832 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:19,832 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:19,832 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. after waiting 0 ms 2024-11-17T08:16:19,833 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:19,833 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:19,833 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c34c3478acc81ba3f3c8f5caec36c13b: Waiting for close lock at 1731831379832Disabling compacts and flushes for region at 1731831379832Disabling writes for close at 1731831379832Writing region close event to WAL at 1731831379833 (+1 ms)Closed at 1731831379833 2024-11-17T08:16:19,835 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T08:16:19,835 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731831379835"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731831379835"}]},"ts":"1731831379835"} 2024-11-17T08:16:19,838 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T08:16:19,839 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T08:16:19,839 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731831379839"}]},"ts":"1731831379839"} 2024-11-17T08:16:19,841 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T08:16:19,842 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c34c3478acc81ba3f3c8f5caec36c13b, ASSIGN}] 2024-11-17T08:16:19,843 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c34c3478acc81ba3f3c8f5caec36c13b, ASSIGN 2024-11-17T08:16:19,844 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c34c3478acc81ba3f3c8f5caec36c13b, ASSIGN; state=OFFLINE, location=e3d0adb87c01,40631,1731831247805; forceNewPlan=false, retain=false 2024-11-17T08:16:19,995 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=c34c3478acc81ba3f3c8f5caec36c13b, regionState=OPENING, regionLocation=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:19,998 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c34c3478acc81ba3f3c8f5caec36c13b, ASSIGN because future has completed 2024-11-17T08:16:19,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805}] 2024-11-17T08:16:20,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-17T08:16:20,158 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:20,158 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7752): Opening region: {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} 2024-11-17T08:16:20,160 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c34c3478acc81ba3f3c8f5caec36c13b 2024-11-17T08:16:20,160 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T08:16:20,160 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7794): checking encryption for c34c3478acc81ba3f3c8f5caec36c13b 2024-11-17T08:16:20,160 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7797): checking classloading for c34c3478acc81ba3f3c8f5caec36c13b 2024-11-17T08:16:20,163 INFO [StoreOpener-c34c3478acc81ba3f3c8f5caec36c13b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c34c3478acc81ba3f3c8f5caec36c13b 2024-11-17T08:16:20,165 INFO [StoreOpener-c34c3478acc81ba3f3c8f5caec36c13b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T08:16:20,165 INFO [StoreOpener-c34c3478acc81ba3f3c8f5caec36c13b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c34c3478acc81ba3f3c8f5caec36c13b columnFamilyName A 2024-11-17T08:16:20,166 DEBUG [StoreOpener-c34c3478acc81ba3f3c8f5caec36c13b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:20,166 INFO [StoreOpener-c34c3478acc81ba3f3c8f5caec36c13b-1 {}] regionserver.HStore(327): Store=c34c3478acc81ba3f3c8f5caec36c13b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T08:16:20,167 INFO [StoreOpener-c34c3478acc81ba3f3c8f5caec36c13b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c34c3478acc81ba3f3c8f5caec36c13b 2024-11-17T08:16:20,169 INFO [StoreOpener-c34c3478acc81ba3f3c8f5caec36c13b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T08:16:20,169 INFO [StoreOpener-c34c3478acc81ba3f3c8f5caec36c13b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c34c3478acc81ba3f3c8f5caec36c13b columnFamilyName B 2024-11-17T08:16:20,169 DEBUG [StoreOpener-c34c3478acc81ba3f3c8f5caec36c13b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:20,170 INFO [StoreOpener-c34c3478acc81ba3f3c8f5caec36c13b-1 {}] regionserver.HStore(327): Store=c34c3478acc81ba3f3c8f5caec36c13b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T08:16:20,170 INFO [StoreOpener-c34c3478acc81ba3f3c8f5caec36c13b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c34c3478acc81ba3f3c8f5caec36c13b 2024-11-17T08:16:20,172 INFO [StoreOpener-c34c3478acc81ba3f3c8f5caec36c13b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T08:16:20,172 INFO [StoreOpener-c34c3478acc81ba3f3c8f5caec36c13b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c34c3478acc81ba3f3c8f5caec36c13b columnFamilyName C 2024-11-17T08:16:20,172 DEBUG [StoreOpener-c34c3478acc81ba3f3c8f5caec36c13b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:20,173 INFO [StoreOpener-c34c3478acc81ba3f3c8f5caec36c13b-1 {}] regionserver.HStore(327): Store=c34c3478acc81ba3f3c8f5caec36c13b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T08:16:20,174 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:20,174 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1038): replaying wal for c34c3478acc81ba3f3c8f5caec36c13b 2024-11-17T08:16:20,175 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b 2024-11-17T08:16:20,175 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b 2024-11-17T08:16:20,176 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1048): stopping wal replay for c34c3478acc81ba3f3c8f5caec36c13b 2024-11-17T08:16:20,176 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1060): Cleaning up temporary data for c34c3478acc81ba3f3c8f5caec36c13b 2024-11-17T08:16:20,177 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T08:16:20,178 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1093): writing seq id for c34c3478acc81ba3f3c8f5caec36c13b 2024-11-17T08:16:20,180 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T08:16:20,181 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1114): Opened c34c3478acc81ba3f3c8f5caec36c13b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72919025, jitterRate=0.08657814562320709}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T08:16:20,181 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c34c3478acc81ba3f3c8f5caec36c13b 2024-11-17T08:16:20,181 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1006): Region open journal for c34c3478acc81ba3f3c8f5caec36c13b: Running coprocessor pre-open hook at 1731831380161Writing region info on filesystem at 1731831380161Initializing all the Stores at 1731831380162 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731831380162Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731831380162Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731831380163 (+1 ms)Cleaning up temporary data from old regions at 1731831380176 (+13 ms)Running coprocessor post-open hooks at 1731831380181 (+5 ms)Region opened successfully at 1731831380181 2024-11-17T08:16:20,182 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., pid=122, masterSystemTime=1731831380152 2024-11-17T08:16:20,184 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:20,184 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:20,184 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=c34c3478acc81ba3f3c8f5caec36c13b, regionState=OPEN, openSeqNum=2, regionLocation=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:20,186 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 because future has completed 2024-11-17T08:16:20,189 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-17T08:16:20,189 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; OpenRegionProcedure c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 in 188 msec 2024-11-17T08:16:20,192 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-11-17T08:16:20,192 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c34c3478acc81ba3f3c8f5caec36c13b, ASSIGN in 347 msec 2024-11-17T08:16:20,193 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T08:16:20,193 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731831380193"}]},"ts":"1731831380193"} 2024-11-17T08:16:20,195 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T08:16:20,196 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T08:16:20,198 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1840 sec 2024-11-17T08:16:21,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-17T08:16:21,176 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:16:21,178 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28ad78a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:21,178 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:16:21,178 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:16:21,180 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:16:21,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:16:21,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:16:21,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22be4046, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:21,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:16:21,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:16:21,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:21,183 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:16:21,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32898172, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:21,184 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:16:21,185 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:16:21,185 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:16:21,187 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:16:21,188 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T08:16:21,189 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is e3d0adb87c01,45089,1731831246985 2024-11-17T08:16:21,189 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@29e822ca 2024-11-17T08:16:21,189 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T08:16:21,190 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T08:16:21,192 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac69b7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:21,192 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:16:21,192 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:16:21,193 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:16:21,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:16:21,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:16:21,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5676c07d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:21,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:16:21,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:16:21,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:21,194 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40604, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:16:21,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@309348e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:21,196 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@277a5c68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:21,196 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:16:21,196 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:16:21,196 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:16:21,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:16:21,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:16:21,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73a583ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:21,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:16:21,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:16:21,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:21,198 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40630, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:16:21,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26bebd9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:21,200 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df5cdc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:21,200 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:16:21,200 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:16:21,201 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:16:21,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:16:21,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:16:21,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61d9636e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:21,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:16:21,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:16:21,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:21,203 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:16:21,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b13617a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:21,207 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b9b902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:21,207 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:16:21,207 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:16:21,208 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:16:21,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:16:21,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:16:21,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61295b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:21,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:16:21,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:16:21,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:21,210 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:16:21,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0a6fce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:21,212 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ff7674f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:21,212 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:16:21,212 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:16:21,213 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:16:21,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:16:21,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:16:21,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14279e38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:21,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:16:21,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:16:21,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:21,214 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:16:21,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60813ae2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:21,216 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@107467c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:21,216 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:16:21,216 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:16:21,217 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:16:21,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:16:21,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:16:21,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ce3ea0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:21,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:16:21,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:16:21,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:21,218 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40676, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:16:21,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d9438fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:21,228 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2acfd5e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:21,228 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:16:21,229 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:16:21,229 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:16:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:16:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:16:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f8d324f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:16:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:16:21,231 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:21,231 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:16:21,232 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61d27bc2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:21,233 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@379767c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:21,233 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:16:21,233 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:16:21,234 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:16:21,234 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:16:21,234 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:16:21,234 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d68b196, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:21,234 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:16:21,234 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:16:21,235 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:21,236 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:16:21,236 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b72f36a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:21,244 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f15306, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:21,244 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:16:21,244 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:16:21,245 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:16:21,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:16:21,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:16:21,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1278b4a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:21,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:16:21,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:16:21,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:21,247 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:16:21,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b79ae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:21,248 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4df5a2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:21,248 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:16:21,248 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:16:21,249 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:16:21,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:16:21,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:16:21,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@644e248b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:21,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:16:21,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:16:21,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:21,250 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:16:21,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ca878de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:21,251 DEBUG [Thread-1988 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:16:21,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:16:21,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:16:21,252 DEBUG [Thread-1999 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:16:21,252 DEBUG [Thread-1992 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:16:21,253 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:16:21,253 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:16:21,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:16:21,253 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:16:21,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:16:21,254 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33874, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:16:21,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2] 2024-11-17T08:16:21,254 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33876, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:16:21,255 DEBUG [Thread-1994 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:16:21,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2] 2024-11-17T08:16:21,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:16:21,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2] 2024-11-17T08:16:21,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:16:21,257 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:16:21,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2] 2024-11-17T08:16:21,259 DEBUG [Thread-2005 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:16:21,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:16:21,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:16:21,262 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33902, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:16:21,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2] 2024-11-17T08:16:21,264 DEBUG [Thread-2003 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:16:21,264 DEBUG [Thread-2001 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:16:21,265 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:16:21,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:16:21,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:16:21,265 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:16:21,266 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33914, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:16:21,266 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33912, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:16:21,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2] 2024-11-17T08:16:21,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2] 2024-11-17T08:16:21,272 DEBUG [Thread-1986 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:16:21,273 DEBUG [Thread-1990 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:16:21,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:16:21,273 DEBUG [Thread-1997 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:16:21,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:16:21,273 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:16:21,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:16:21,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:16:21,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:16:21,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:16:21,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-17T08:16:21,275 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33940, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:16:21,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T08:16:21,275 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:16:21,275 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33930, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:16:21,275 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33944, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:16:21,276 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:16:21,276 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:16:21,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2] 2024-11-17T08:16:21,276 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2] 2024-11-17T08:16:21,277 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2] 2024-11-17T08:16:21,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on c34c3478acc81ba3f3c8f5caec36c13b 2024-11-17T08:16:21,296 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c34c3478acc81ba3f3c8f5caec36c13b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-17T08:16:21,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=A 2024-11-17T08:16:21,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:21,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=B 2024-11-17T08:16:21,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:21,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=C 2024-11-17T08:16:21,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:21,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/c87b5052e8234c21bdbf3da7f3b8e178 is 50, key is test_row_0/A:col10/1731831381295/Put/seqid=0 2024-11-17T08:16:21,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742266_1442 (size=14337) 2024-11-17T08:16:21,320 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/c87b5052e8234c21bdbf3da7f3b8e178 2024-11-17T08:16:21,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:21,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831441317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:21,323 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731831381309, max timestamp=1731831381320Num uniques 859; 2024-11-17T08:16:21,323 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-17T08:16:21,323 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731831381309, max timestamp=1731831381320Num uniques 859; 2024-11-17T08:16:21,323 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-17T08:16:21,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:21,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:21,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:21,325 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731831381312, max timestamp=1731831381320Num uniques 859; 2024-11-17T08:16:21,325 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T08:16:21,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:21,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831441344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:21,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:21,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:21,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:21,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:21,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33872 deadline: 1731831441345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:21,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:21,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:21,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:21,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/7222f61b3885491299633857026bf526 is 50, key is test_row_0/B:col10/1731831381295/Put/seqid=0 2024-11-17T08:16:21,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:21,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831441346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:21,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742267_1443 (size=9657) 2024-11-17T08:16:21,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:21,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33940 deadline: 1731831441346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:21,357 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:21,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T08:16:21,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T08:16:21,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:21,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. as already flushing 2024-11-17T08:16:21,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:21,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:21,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:21,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:21,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:21,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831441433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:21,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:21,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:21,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:21,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:21,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831441453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:21,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:21,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:21,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33872 deadline: 1731831441453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:21,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:21,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:21,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:21,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:21,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:21,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:21,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831441464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:21,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:21,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:21,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:21,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33940 deadline: 1731831441464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:21,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:21,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:21,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:21,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:21,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T08:16:21,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:21,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. as already flushing 2024-11-17T08:16:21,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:21,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:21,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:21,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:21,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T08:16:21,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:21,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831441643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:21,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:21,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:21,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:21,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831441664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:21,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:21,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:21,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:21,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:21,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33872 deadline: 1731831441664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:21,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:21,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:21,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:21,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:21,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831441674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:21,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:21,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:21,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:21,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:21,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33940 deadline: 1731831441674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:21,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:21,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:21,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:21,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T08:16:21,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:21,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. as already flushing 2024-11-17T08:16:21,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:21,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:21,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:21,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:21,757 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/7222f61b3885491299633857026bf526 2024-11-17T08:16:21,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/2055e66129ad4be9a3b43582f6c571f1 is 50, key is test_row_0/C:col10/1731831381295/Put/seqid=0 2024-11-17T08:16:21,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742268_1444 (size=9657) 2024-11-17T08:16:21,784 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/2055e66129ad4be9a3b43582f6c571f1 2024-11-17T08:16:21,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/c87b5052e8234c21bdbf3da7f3b8e178 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/c87b5052e8234c21bdbf3da7f3b8e178 2024-11-17T08:16:21,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/c87b5052e8234c21bdbf3da7f3b8e178, entries=200, sequenceid=12, filesize=14.0 K 2024-11-17T08:16:21,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/7222f61b3885491299633857026bf526 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/7222f61b3885491299633857026bf526 2024-11-17T08:16:21,797 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/7222f61b3885491299633857026bf526, entries=100, sequenceid=12, filesize=9.4 K 2024-11-17T08:16:21,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/2055e66129ad4be9a3b43582f6c571f1 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/2055e66129ad4be9a3b43582f6c571f1 2024-11-17T08:16:21,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/2055e66129ad4be9a3b43582f6c571f1, entries=100, sequenceid=12, filesize=9.4 K 2024-11-17T08:16:21,801 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=129.71 KB/132820 for c34c3478acc81ba3f3c8f5caec36c13b in 505ms, sequenceid=12, compaction requested=false 2024-11-17T08:16:21,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c34c3478acc81ba3f3c8f5caec36c13b: 2024-11-17T08:16:21,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T08:16:21,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:21,887 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing c34c3478acc81ba3f3c8f5caec36c13b 3/3 column families, dataSize=129.71 KB heapSize=356.08 KB 2024-11-17T08:16:21,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=A 2024-11-17T08:16:21,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:16:21,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=B 2024-11-17T08:16:21,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:16:21,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=C 2024-11-17T08:16:21,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:16:21,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/0199baa9df41480fb40b5902761ec907 is 50, key is test_row_0/A:col10/1731831381343/Put/seqid=0 2024-11-17T08:16:21,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742269_1445 (size=12001) 2024-11-17T08:16:21,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T08:16:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on c34c3478acc81ba3f3c8f5caec36c13b 2024-11-17T08:16:21,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. as already flushing 2024-11-17T08:16:21,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:21,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33872 deadline: 1731831441982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:21,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:21,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:21,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:21,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831441984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:21,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:21,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:21,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33940 deadline: 1731831441984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:21,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:21,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:21,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:21,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831441987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:21,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:21,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:21,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:21,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:22,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:22,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831441991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:22,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:22,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:22,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:22,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:22,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33872 deadline: 1731831442094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:22,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:22,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:22,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:22,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:22,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831442103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:22,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:22,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:22,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831442103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:22,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:22,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:22,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:22,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:22,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:22,296 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/0199baa9df41480fb40b5902761ec907 2024-11-17T08:16:22,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/ed3f75cff5274e848a8424d4a4bfb511 is 50, key is test_row_0/B:col10/1731831381343/Put/seqid=0 2024-11-17T08:16:22,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742270_1446 (size=12001) 2024-11-17T08:16:22,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:22,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33872 deadline: 1731831442304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:22,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:22,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:22,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:22,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831442313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:22,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:22,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831442313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:22,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:22,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:22,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:22,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:22,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T08:16:22,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831442504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:22,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:22,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:22,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33940 deadline: 1731831442504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:22,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:22,511 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:22,511 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:22,511 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:22,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33872 deadline: 1731831442614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:22,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:22,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:22,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:22,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831442633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:22,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:22,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:22,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:22,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831442633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:22,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:22,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:22,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:22,706 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/ed3f75cff5274e848a8424d4a4bfb511 2024-11-17T08:16:22,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/1436fc9320964b87b05ba3badc6c137a is 50, key is test_row_0/C:col10/1731831381343/Put/seqid=0 2024-11-17T08:16:22,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742271_1447 (size=12001) 2024-11-17T08:16:22,714 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/1436fc9320964b87b05ba3badc6c137a 2024-11-17T08:16:22,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/0199baa9df41480fb40b5902761ec907 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/0199baa9df41480fb40b5902761ec907 2024-11-17T08:16:22,720 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/0199baa9df41480fb40b5902761ec907, entries=150, sequenceid=45, filesize=11.7 K 2024-11-17T08:16:22,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/ed3f75cff5274e848a8424d4a4bfb511 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/ed3f75cff5274e848a8424d4a4bfb511 2024-11-17T08:16:22,723 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/ed3f75cff5274e848a8424d4a4bfb511, entries=150, sequenceid=45, filesize=11.7 K 2024-11-17T08:16:22,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/1436fc9320964b87b05ba3badc6c137a as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/1436fc9320964b87b05ba3badc6c137a 2024-11-17T08:16:22,728 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/1436fc9320964b87b05ba3badc6c137a, entries=150, sequenceid=45, filesize=11.7 K 2024-11-17T08:16:22,728 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~356.03 KB/364576, currentSize=53.67 KB/54960 for c34c3478acc81ba3f3c8f5caec36c13b in 841ms, sequenceid=45, compaction requested=false 2024-11-17T08:16:22,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for c34c3478acc81ba3f3c8f5caec36c13b: 2024-11-17T08:16:22,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:22,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-17T08:16:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-17T08:16:22,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-17T08:16:22,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4540 sec 2024-11-17T08:16:22,732 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.4580 sec 2024-11-17T08:16:23,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on c34c3478acc81ba3f3c8f5caec36c13b 2024-11-17T08:16:23,137 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c34c3478acc81ba3f3c8f5caec36c13b 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-17T08:16:23,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=A 2024-11-17T08:16:23,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:23,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=B 2024-11-17T08:16:23,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:23,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=C 2024-11-17T08:16:23,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:23,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/9b3916429b2d4169a12547d9dac4c7e1 is 50, key is test_row_0/A:col10/1731831381980/Put/seqid=0 2024-11-17T08:16:23,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742272_1448 (size=14341) 2024-11-17T08:16:23,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:23,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831443196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:23,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:23,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:23,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:23,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:23,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33872 deadline: 1731831443198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:23,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:23,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:23,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:23,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:23,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831443199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:23,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:23,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:23,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:23,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:23,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831443313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:23,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:23,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33872 deadline: 1731831443313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:23,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:23,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:23,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:23,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:23,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:23,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:23,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:23,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831443314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:23,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:23,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:23,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:23,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T08:16:23,424 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:16:23,424 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:16:23,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-17T08:16:23,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T08:16:23,425 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:16:23,426 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:16:23,426 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:16:23,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:23,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831443524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:23,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:23,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:23,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33872 deadline: 1731831443524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:23,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:23,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:23,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:23,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:23,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:23,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:23,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831443524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:23,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:23,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831443524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:23,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:23,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:23,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:23,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:23,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33940 deadline: 1731831443524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:23,532 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to e3d0adb87c01,40631,1731831247805 for 'test_row_2' in c34c3478acc81ba3f3c8f5caec36c13b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T08:16:23,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:23,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:23,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:23,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T08:16:23,533 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to e3d0adb87c01,40631,1731831247805 for 'test_row_2' in c34c3478acc81ba3f3c8f5caec36c13b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T08:16:23,533 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:23,533 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:23,534 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:23,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/9b3916429b2d4169a12547d9dac4c7e1 2024-11-17T08:16:23,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/358af8cd766e4ea5bead05d441f94556 is 50, key is test_row_0/B:col10/1731831381980/Put/seqid=0 2024-11-17T08:16:23,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742273_1449 (size=12001) 2024-11-17T08:16:23,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T08:16:23,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:23,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. as already flushing 2024-11-17T08:16:23,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:23,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:23,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:23,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:23,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T08:16:23,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:23,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. as already flushing 2024-11-17T08:16:23,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:23,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:23,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:23,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:23,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T08:16:23,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:23,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831443843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:23,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:23,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:23,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33872 deadline: 1731831443843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:23,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:23,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:23,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:23,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:23,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:23,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831443843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:23,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:23,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:23,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:23,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:23,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T08:16:23,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:23,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. as already flushing 2024-11-17T08:16:23,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:23,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:23,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:23,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:23,954 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/358af8cd766e4ea5bead05d441f94556 2024-11-17T08:16:23,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/b36c2e7b3f20448faea602af57198f6b is 50, key is test_row_0/C:col10/1731831381980/Put/seqid=0 2024-11-17T08:16:23,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742274_1450 (size=12001) 2024-11-17T08:16:23,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/b36c2e7b3f20448faea602af57198f6b 2024-11-17T08:16:23,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/9b3916429b2d4169a12547d9dac4c7e1 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/9b3916429b2d4169a12547d9dac4c7e1 2024-11-17T08:16:23,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/9b3916429b2d4169a12547d9dac4c7e1, entries=200, sequenceid=57, filesize=14.0 K 2024-11-17T08:16:23,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/358af8cd766e4ea5bead05d441f94556 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/358af8cd766e4ea5bead05d441f94556 2024-11-17T08:16:23,985 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/358af8cd766e4ea5bead05d441f94556, entries=150, sequenceid=57, filesize=11.7 K 2024-11-17T08:16:23,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/b36c2e7b3f20448faea602af57198f6b as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/b36c2e7b3f20448faea602af57198f6b 2024-11-17T08:16:23,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/b36c2e7b3f20448faea602af57198f6b, entries=150, sequenceid=57, filesize=11.7 K 2024-11-17T08:16:23,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c34c3478acc81ba3f3c8f5caec36c13b in 856ms, sequenceid=57, compaction requested=true 2024-11-17T08:16:23,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c34c3478acc81ba3f3c8f5caec36c13b: 2024-11-17T08:16:23,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c34c3478acc81ba3f3c8f5caec36c13b:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:16:23,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:23,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c34c3478acc81ba3f3c8f5caec36c13b:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:16:23,994 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:16:23,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:23,994 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:16:23,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c34c3478acc81ba3f3c8f5caec36c13b:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:16:23,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:16:23,995 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:16:23,995 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): c34c3478acc81ba3f3c8f5caec36c13b/B is initiating minor compaction (all files) 2024-11-17T08:16:23,995 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c34c3478acc81ba3f3c8f5caec36c13b/B in TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:23,995 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/7222f61b3885491299633857026bf526, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/ed3f75cff5274e848a8424d4a4bfb511, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/358af8cd766e4ea5bead05d441f94556] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp, totalSize=32.9 K 2024-11-17T08:16:23,995 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40679 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:16:23,995 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): c34c3478acc81ba3f3c8f5caec36c13b/A is initiating minor compaction (all files) 2024-11-17T08:16:23,995 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c34c3478acc81ba3f3c8f5caec36c13b/A in TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:23,995 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/c87b5052e8234c21bdbf3da7f3b8e178, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/0199baa9df41480fb40b5902761ec907, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/9b3916429b2d4169a12547d9dac4c7e1] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp, totalSize=39.7 K 2024-11-17T08:16:23,995 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 7222f61b3885491299633857026bf526, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731831381295 2024-11-17T08:16:23,995 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting c87b5052e8234c21bdbf3da7f3b8e178, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731831381257 2024-11-17T08:16:23,995 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting ed3f75cff5274e848a8424d4a4bfb511, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731831381342 2024-11-17T08:16:23,996 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0199baa9df41480fb40b5902761ec907, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731831381342 2024-11-17T08:16:23,996 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 358af8cd766e4ea5bead05d441f94556, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731831381980 2024-11-17T08:16:23,996 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b3916429b2d4169a12547d9dac4c7e1, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731831381980 2024-11-17T08:16:24,002 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c34c3478acc81ba3f3c8f5caec36c13b#A#compaction#375 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:24,002 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c34c3478acc81ba3f3c8f5caec36c13b#B#compaction#376 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:24,003 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/467b501d7a4a4c5fb118e9cf93147720 is 50, key is test_row_0/A:col10/1731831381980/Put/seqid=0 2024-11-17T08:16:24,003 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/895e94abfb53409cab7d81e41c33a377 is 50, key is test_row_0/B:col10/1731831381980/Put/seqid=0 2024-11-17T08:16:24,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742275_1451 (size=12104) 2024-11-17T08:16:24,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742276_1452 (size=12104) 2024-11-17T08:16:24,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T08:16:24,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:24,036 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing c34c3478acc81ba3f3c8f5caec36c13b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-17T08:16:24,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=A 2024-11-17T08:16:24,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:24,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=B 2024-11-17T08:16:24,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:24,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=C 2024-11-17T08:16:24,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:24,038 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/467b501d7a4a4c5fb118e9cf93147720 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/467b501d7a4a4c5fb118e9cf93147720 2024-11-17T08:16:24,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/bf7e3059796e4cf7b082026fb25a7585 is 50, key is test_row_0/A:col10/1731831383198/Put/seqid=0 2024-11-17T08:16:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T08:16:24,056 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c34c3478acc81ba3f3c8f5caec36c13b/A of c34c3478acc81ba3f3c8f5caec36c13b into 467b501d7a4a4c5fb118e9cf93147720(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:24,056 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c34c3478acc81ba3f3c8f5caec36c13b: 2024-11-17T08:16:24,056 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., storeName=c34c3478acc81ba3f3c8f5caec36c13b/A, priority=13, startTime=1731831383993; duration=0sec 2024-11-17T08:16:24,058 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:16:24,058 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c34c3478acc81ba3f3c8f5caec36c13b:A 2024-11-17T08:16:24,058 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:16:24,060 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:16:24,061 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): c34c3478acc81ba3f3c8f5caec36c13b/C is initiating minor compaction (all files) 2024-11-17T08:16:24,061 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c34c3478acc81ba3f3c8f5caec36c13b/C in TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:24,061 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/2055e66129ad4be9a3b43582f6c571f1, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/1436fc9320964b87b05ba3badc6c137a, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/b36c2e7b3f20448faea602af57198f6b] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp, totalSize=32.9 K 2024-11-17T08:16:24,061 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2055e66129ad4be9a3b43582f6c571f1, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731831381295 2024-11-17T08:16:24,061 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1436fc9320964b87b05ba3badc6c137a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731831381342 2024-11-17T08:16:24,062 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting b36c2e7b3f20448faea602af57198f6b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731831381980 2024-11-17T08:16:24,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742277_1453 (size=12001) 2024-11-17T08:16:24,094 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c34c3478acc81ba3f3c8f5caec36c13b#C#compaction#378 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:24,095 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/d83fda56faf449698065fdc39efe2bd7 is 50, key is test_row_0/C:col10/1731831381980/Put/seqid=0 2024-11-17T08:16:24,121 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T08:16:24,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742278_1454 (size=12104) 2024-11-17T08:16:24,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on c34c3478acc81ba3f3c8f5caec36c13b 2024-11-17T08:16:24,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. as already flushing 2024-11-17T08:16:24,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:24,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831444373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:24,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:24,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:24,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:24,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831444374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:24,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:24,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:24,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:24,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:24,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:24,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33872 deadline: 1731831444376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:24,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:24,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:24,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:24,414 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/895e94abfb53409cab7d81e41c33a377 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/895e94abfb53409cab7d81e41c33a377 2024-11-17T08:16:24,417 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c34c3478acc81ba3f3c8f5caec36c13b/B of c34c3478acc81ba3f3c8f5caec36c13b into 895e94abfb53409cab7d81e41c33a377(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:24,418 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c34c3478acc81ba3f3c8f5caec36c13b: 2024-11-17T08:16:24,418 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., storeName=c34c3478acc81ba3f3c8f5caec36c13b/B, priority=13, startTime=1731831383994; duration=0sec 2024-11-17T08:16:24,418 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:24,418 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c34c3478acc81ba3f3c8f5caec36c13b:B 2024-11-17T08:16:24,478 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/bf7e3059796e4cf7b082026fb25a7585 2024-11-17T08:16:24,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/70a753a14d214953bcaf2b6f92cec342 is 50, key is test_row_0/B:col10/1731831383198/Put/seqid=0 2024-11-17T08:16:24,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742279_1455 (size=12001) 2024-11-17T08:16:24,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:24,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831444484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:24,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:24,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:24,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831444484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:24,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:24,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:24,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:24,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:24,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:24,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:24,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33872 deadline: 1731831444493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:24,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:24,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:24,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:24,542 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/d83fda56faf449698065fdc39efe2bd7 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/d83fda56faf449698065fdc39efe2bd7 2024-11-17T08:16:24,545 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c34c3478acc81ba3f3c8f5caec36c13b/C of c34c3478acc81ba3f3c8f5caec36c13b into d83fda56faf449698065fdc39efe2bd7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:24,545 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c34c3478acc81ba3f3c8f5caec36c13b: 2024-11-17T08:16:24,545 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., storeName=c34c3478acc81ba3f3c8f5caec36c13b/C, priority=13, startTime=1731831383994; duration=0sec 2024-11-17T08:16:24,545 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:24,545 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c34c3478acc81ba3f3c8f5caec36c13b:C 2024-11-17T08:16:24,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T08:16:24,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:24,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831444694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:24,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:24,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:24,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:24,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:24,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831444704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:24,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:24,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:24,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:24,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:24,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33872 deadline: 1731831444704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:24,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:24,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:24,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:24,888 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/70a753a14d214953bcaf2b6f92cec342 2024-11-17T08:16:24,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/685726a7a8d349ccb24ed373c9099649 is 50, key is test_row_0/C:col10/1731831383198/Put/seqid=0 2024-11-17T08:16:24,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742280_1456 (size=12001) 2024-11-17T08:16:25,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:25,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831445004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:25,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:25,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:25,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:25,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:25,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831445014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:25,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:25,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:25,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:25,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33872 deadline: 1731831445014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:25,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:25,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:25,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:25,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:25,297 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/685726a7a8d349ccb24ed373c9099649 2024-11-17T08:16:25,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/bf7e3059796e4cf7b082026fb25a7585 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/bf7e3059796e4cf7b082026fb25a7585 2024-11-17T08:16:25,303 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/bf7e3059796e4cf7b082026fb25a7585, entries=150, sequenceid=77, filesize=11.7 K 2024-11-17T08:16:25,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/70a753a14d214953bcaf2b6f92cec342 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/70a753a14d214953bcaf2b6f92cec342 2024-11-17T08:16:25,306 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/70a753a14d214953bcaf2b6f92cec342, entries=150, sequenceid=77, filesize=11.7 K 2024-11-17T08:16:25,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/685726a7a8d349ccb24ed373c9099649 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/685726a7a8d349ccb24ed373c9099649 2024-11-17T08:16:25,310 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/685726a7a8d349ccb24ed373c9099649, entries=150, sequenceid=77, filesize=11.7 K 2024-11-17T08:16:25,311 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c34c3478acc81ba3f3c8f5caec36c13b in 1275ms, sequenceid=77, compaction requested=false 2024-11-17T08:16:25,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for c34c3478acc81ba3f3c8f5caec36c13b: 2024-11-17T08:16:25,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:25,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-17T08:16:25,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-17T08:16:25,314 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-17T08:16:25,314 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8860 sec 2024-11-17T08:16:25,315 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.8900 sec 2024-11-17T08:16:25,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on c34c3478acc81ba3f3c8f5caec36c13b 2024-11-17T08:16:25,518 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c34c3478acc81ba3f3c8f5caec36c13b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-17T08:16:25,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=A 2024-11-17T08:16:25,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:25,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=B 2024-11-17T08:16:25,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:25,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=C 2024-11-17T08:16:25,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:25,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/73c000c521d74973b1a43a3311313aed is 50, key is test_row_0/A:col10/1731831384375/Put/seqid=0 2024-11-17T08:16:25,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742281_1457 (size=14341) 2024-11-17T08:16:25,528 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/73c000c521d74973b1a43a3311313aed 2024-11-17T08:16:25,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/00145a702f9a4af9b87a1445bae4d11a is 50, key is test_row_0/B:col10/1731831384375/Put/seqid=0 2024-11-17T08:16:25,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742282_1458 (size=12001) 2024-11-17T08:16:25,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/00145a702f9a4af9b87a1445bae4d11a 2024-11-17T08:16:25,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/b6579b248233456f9c677f4b0c7ab378 is 50, key is test_row_0/C:col10/1731831384375/Put/seqid=0 2024-11-17T08:16:25,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742283_1459 (size=12001) 2024-11-17T08:16:25,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/b6579b248233456f9c677f4b0c7ab378 2024-11-17T08:16:25,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/73c000c521d74973b1a43a3311313aed as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/73c000c521d74973b1a43a3311313aed 2024-11-17T08:16:25,571 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/73c000c521d74973b1a43a3311313aed, entries=200, sequenceid=93, filesize=14.0 K 2024-11-17T08:16:25,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/00145a702f9a4af9b87a1445bae4d11a as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/00145a702f9a4af9b87a1445bae4d11a 2024-11-17T08:16:25,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T08:16:25,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/00145a702f9a4af9b87a1445bae4d11a, entries=150, sequenceid=93, filesize=11.7 K 2024-11-17T08:16:25,574 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:16:25,575 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:16:25,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/b6579b248233456f9c677f4b0c7ab378 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/b6579b248233456f9c677f4b0c7ab378 2024-11-17T08:16:25,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-17T08:16:25,577 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:16:25,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T08:16:25,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:25,577 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:16:25,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831445563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:25,578 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:16:25,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:25,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:25,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:25,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/b6579b248233456f9c677f4b0c7ab378, entries=150, sequenceid=93, filesize=11.7 K 2024-11-17T08:16:25,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:25,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831445567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:25,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:25,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:25,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:25,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for c34c3478acc81ba3f3c8f5caec36c13b in 62ms, sequenceid=93, compaction requested=true 2024-11-17T08:16:25,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c34c3478acc81ba3f3c8f5caec36c13b: 2024-11-17T08:16:25,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on c34c3478acc81ba3f3c8f5caec36c13b 2024-11-17T08:16:25,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c34c3478acc81ba3f3c8f5caec36c13b:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:16:25,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:25,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c34c3478acc81ba3f3c8f5caec36c13b:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:16:25,581 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:16:25,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:25,581 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:16:25,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c34c3478acc81ba3f3c8f5caec36c13b:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:16:25,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:16:25,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c34c3478acc81ba3f3c8f5caec36c13b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-17T08:16:25,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=A 2024-11-17T08:16:25,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:16:25,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=B 2024-11-17T08:16:25,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:16:25,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=C 2024-11-17T08:16:25,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:16:25,584 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:16:25,584 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:16:25,584 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): c34c3478acc81ba3f3c8f5caec36c13b/A is initiating minor compaction (all files) 2024-11-17T08:16:25,584 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): c34c3478acc81ba3f3c8f5caec36c13b/B is initiating minor compaction (all files) 2024-11-17T08:16:25,584 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c34c3478acc81ba3f3c8f5caec36c13b/A in TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:25,584 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/467b501d7a4a4c5fb118e9cf93147720, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/bf7e3059796e4cf7b082026fb25a7585, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/73c000c521d74973b1a43a3311313aed] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp, totalSize=37.5 K 2024-11-17T08:16:25,584 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c34c3478acc81ba3f3c8f5caec36c13b/B in TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:25,584 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/895e94abfb53409cab7d81e41c33a377, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/70a753a14d214953bcaf2b6f92cec342, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/00145a702f9a4af9b87a1445bae4d11a] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp, totalSize=35.3 K 2024-11-17T08:16:25,585 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 467b501d7a4a4c5fb118e9cf93147720, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731831381980 2024-11-17T08:16:25,585 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 895e94abfb53409cab7d81e41c33a377, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731831381980 2024-11-17T08:16:25,585 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf7e3059796e4cf7b082026fb25a7585, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731831383194 2024-11-17T08:16:25,586 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73c000c521d74973b1a43a3311313aed, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731831384368 2024-11-17T08:16:25,586 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 70a753a14d214953bcaf2b6f92cec342, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731831383194 2024-11-17T08:16:25,586 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 00145a702f9a4af9b87a1445bae4d11a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731831384372 2024-11-17T08:16:25,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/e9055076ea234762a747f05593e45962 is 50, key is test_row_0/A:col10/1731831385581/Put/seqid=0 2024-11-17T08:16:25,592 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c34c3478acc81ba3f3c8f5caec36c13b#B#compaction#385 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:25,592 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/d8612083b1774a49a9ad83a081d79e14 is 50, key is test_row_0/B:col10/1731831384375/Put/seqid=0 2024-11-17T08:16:25,595 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c34c3478acc81ba3f3c8f5caec36c13b#A#compaction#386 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:25,595 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/39ef1982fd5841d4ab24daba698d255e is 50, key is test_row_0/A:col10/1731831384375/Put/seqid=0 2024-11-17T08:16:25,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742284_1460 (size=16681) 2024-11-17T08:16:25,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742286_1462 (size=12207) 2024-11-17T08:16:25,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742285_1461 (size=12207) 2024-11-17T08:16:25,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:25,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33940 deadline: 1731831445609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:25,623 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:25,623 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:25,623 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:25,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:25,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831445623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:25,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:25,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:25,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:25,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33872 deadline: 1731831445623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:25,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:25,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:25,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:25,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:25,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T08:16:25,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:25,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831445684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:25,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:25,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:25,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:25,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:25,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831445684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:25,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:25,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:25,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:25,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T08:16:25,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:25,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. as already flushing 2024-11-17T08:16:25,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:25,730 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:25,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:25,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:25,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:25,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33940 deadline: 1731831445733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:25,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:25,748 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:25,748 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:25,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831445733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:25,748 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:25,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:25,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:25,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:25,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:25,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33872 deadline: 1731831445733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:25,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:25,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:25,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:25,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T08:16:25,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:25,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. as already flushing 2024-11-17T08:16:25,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:25,883 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:25,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:25,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:25,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T08:16:25,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:25,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831445904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:25,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:25,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:25,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:25,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:25,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831445904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:25,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:25,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:25,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:25,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:25,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33940 deadline: 1731831445954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:25,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:25,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:25,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:25,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:25,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831445954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:25,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:25,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:25,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:25,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:25,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33872 deadline: 1731831445954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:25,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:25,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:25,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:26,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/e9055076ea234762a747f05593e45962 2024-11-17T08:16:26,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/47ce2268ce0247eeadaeff11d9f0a9e7 is 50, key is test_row_0/B:col10/1731831385581/Put/seqid=0 2024-11-17T08:16:26,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742287_1463 (size=12001) 2024-11-17T08:16:26,025 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/39ef1982fd5841d4ab24daba698d255e as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/39ef1982fd5841d4ab24daba698d255e 2024-11-17T08:16:26,029 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/d8612083b1774a49a9ad83a081d79e14 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/d8612083b1774a49a9ad83a081d79e14 2024-11-17T08:16:26,029 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c34c3478acc81ba3f3c8f5caec36c13b/A of c34c3478acc81ba3f3c8f5caec36c13b into 39ef1982fd5841d4ab24daba698d255e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:26,029 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c34c3478acc81ba3f3c8f5caec36c13b: 2024-11-17T08:16:26,029 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., storeName=c34c3478acc81ba3f3c8f5caec36c13b/A, priority=13, startTime=1731831385581; duration=0sec 2024-11-17T08:16:26,029 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:16:26,029 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c34c3478acc81ba3f3c8f5caec36c13b:A 2024-11-17T08:16:26,029 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:16:26,030 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:16:26,030 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): c34c3478acc81ba3f3c8f5caec36c13b/C is initiating minor compaction (all files) 2024-11-17T08:16:26,030 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c34c3478acc81ba3f3c8f5caec36c13b/C in TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:26,030 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/d83fda56faf449698065fdc39efe2bd7, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/685726a7a8d349ccb24ed373c9099649, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/b6579b248233456f9c677f4b0c7ab378] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp, totalSize=35.3 K 2024-11-17T08:16:26,030 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting d83fda56faf449698065fdc39efe2bd7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731831381980 2024-11-17T08:16:26,031 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 685726a7a8d349ccb24ed373c9099649, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731831383194 2024-11-17T08:16:26,031 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6579b248233456f9c677f4b0c7ab378, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731831384372 2024-11-17T08:16:26,033 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c34c3478acc81ba3f3c8f5caec36c13b/B of c34c3478acc81ba3f3c8f5caec36c13b into d8612083b1774a49a9ad83a081d79e14(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:26,033 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c34c3478acc81ba3f3c8f5caec36c13b: 2024-11-17T08:16:26,033 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., storeName=c34c3478acc81ba3f3c8f5caec36c13b/B, priority=13, startTime=1731831385581; duration=0sec 2024-11-17T08:16:26,033 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:26,033 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c34c3478acc81ba3f3c8f5caec36c13b:B 2024-11-17T08:16:26,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T08:16:26,036 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c34c3478acc81ba3f3c8f5caec36c13b#C#compaction#388 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:26,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:26,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. as already flushing 2024-11-17T08:16:26,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:26,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:26,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:26,036 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/c4b35217441c4ce4a6471c7f8ea176e9 is 50, key is test_row_0/C:col10/1731831384375/Put/seqid=0 2024-11-17T08:16:26,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:26,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742288_1464 (size=12207) 2024-11-17T08:16:26,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T08:16:26,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:26,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. as already flushing 2024-11-17T08:16:26,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:26,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:26,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:26,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:26,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T08:16:26,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:26,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831446214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:26,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:26,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:26,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:26,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831446223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:26,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:26,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:26,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:26,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:26,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831446263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:26,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:26,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:26,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:26,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33940 deadline: 1731831446274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:26,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:26,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:26,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33872 deadline: 1731831446274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:26,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:26,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:26,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:26,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:26,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T08:16:26,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:26,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. as already flushing 2024-11-17T08:16:26,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:26,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:26,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:26,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:26,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/47ce2268ce0247eeadaeff11d9f0a9e7 2024-11-17T08:16:26,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/de202e4c093c443da3df448adf077de9 is 50, key is test_row_0/C:col10/1731831385581/Put/seqid=0 2024-11-17T08:16:26,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742289_1465 (size=12001) 2024-11-17T08:16:26,446 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/c4b35217441c4ce4a6471c7f8ea176e9 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/c4b35217441c4ce4a6471c7f8ea176e9 2024-11-17T08:16:26,450 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c34c3478acc81ba3f3c8f5caec36c13b/C of c34c3478acc81ba3f3c8f5caec36c13b into c4b35217441c4ce4a6471c7f8ea176e9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:26,450 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c34c3478acc81ba3f3c8f5caec36c13b: 2024-11-17T08:16:26,450 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., storeName=c34c3478acc81ba3f3c8f5caec36c13b/C, priority=13, startTime=1731831385581; duration=0sec 2024-11-17T08:16:26,450 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:26,450 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c34c3478acc81ba3f3c8f5caec36c13b:C 2024-11-17T08:16:26,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T08:16:26,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:26,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. as already flushing 2024-11-17T08:16:26,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:26,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:26,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:26,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T08:16:26,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:26,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. as already flushing 2024-11-17T08:16:26,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:26,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:26,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:26,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:26,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T08:16:26,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:26,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831446733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:26,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:26,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:26,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:26,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:26,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831446743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:26,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:26,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:26,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:26,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:26,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831446784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:26,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:26,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:26,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:26,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T08:16:26,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:26,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. as already flushing 2024-11-17T08:16:26,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:26,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:26,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:26,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:26,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33940 deadline: 1731831446793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:26,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:26,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:26,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:26,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:26,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33872 deadline: 1731831446793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:26,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:26,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:26,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:26,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:26,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/de202e4c093c443da3df448adf077de9 2024-11-17T08:16:26,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/e9055076ea234762a747f05593e45962 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/e9055076ea234762a747f05593e45962 2024-11-17T08:16:26,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/e9055076ea234762a747f05593e45962, entries=250, sequenceid=114, filesize=16.3 K 2024-11-17T08:16:26,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/47ce2268ce0247eeadaeff11d9f0a9e7 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/47ce2268ce0247eeadaeff11d9f0a9e7 2024-11-17T08:16:26,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/47ce2268ce0247eeadaeff11d9f0a9e7, entries=150, sequenceid=114, filesize=11.7 K 2024-11-17T08:16:26,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/de202e4c093c443da3df448adf077de9 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/de202e4c093c443da3df448adf077de9 2024-11-17T08:16:26,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/de202e4c093c443da3df448adf077de9, entries=150, sequenceid=114, filesize=11.7 K 2024-11-17T08:16:26,843 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for c34c3478acc81ba3f3c8f5caec36c13b in 1262ms, sequenceid=114, compaction requested=false 2024-11-17T08:16:26,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c34c3478acc81ba3f3c8f5caec36c13b: 2024-11-17T08:16:26,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T08:16:26,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:26,954 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing c34c3478acc81ba3f3c8f5caec36c13b 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-17T08:16:26,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=A 2024-11-17T08:16:26,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:26,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=B 2024-11-17T08:16:26,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:26,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=C 2024-11-17T08:16:26,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:26,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/f6e98abedc8c4738abfd3b750295cf01 is 50, key is test_row_0/A:col10/1731831385607/Put/seqid=0 2024-11-17T08:16:26,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742290_1466 (size=12001) 2024-11-17T08:16:27,362 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/f6e98abedc8c4738abfd3b750295cf01 2024-11-17T08:16:27,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/30b1e7adf47b457c8d172d7c841b38e0 is 50, key is test_row_0/B:col10/1731831385607/Put/seqid=0 2024-11-17T08:16:27,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742291_1467 (size=12001) 2024-11-17T08:16:27,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T08:16:27,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on c34c3478acc81ba3f3c8f5caec36c13b 2024-11-17T08:16:27,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. as already flushing 2024-11-17T08:16:27,782 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/30b1e7adf47b457c8d172d7c841b38e0 2024-11-17T08:16:27,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/273f89913f534258ace2097b0e81e67b is 50, key is test_row_0/C:col10/1731831385607/Put/seqid=0 2024-11-17T08:16:27,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742292_1468 (size=12001) 2024-11-17T08:16:27,805 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/273f89913f534258ace2097b0e81e67b 2024-11-17T08:16:27,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/f6e98abedc8c4738abfd3b750295cf01 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/f6e98abedc8c4738abfd3b750295cf01 2024-11-17T08:16:27,811 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/f6e98abedc8c4738abfd3b750295cf01, entries=150, sequenceid=129, filesize=11.7 K 2024-11-17T08:16:27,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/30b1e7adf47b457c8d172d7c841b38e0 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/30b1e7adf47b457c8d172d7c841b38e0 2024-11-17T08:16:27,814 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/30b1e7adf47b457c8d172d7c841b38e0, entries=150, sequenceid=129, filesize=11.7 K 2024-11-17T08:16:27,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/273f89913f534258ace2097b0e81e67b as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/273f89913f534258ace2097b0e81e67b 2024-11-17T08:16:27,817 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/273f89913f534258ace2097b0e81e67b, entries=150, sequenceid=129, filesize=11.7 K 2024-11-17T08:16:27,818 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=107.34 KB/109920 for c34c3478acc81ba3f3c8f5caec36c13b in 864ms, sequenceid=129, compaction requested=true 2024-11-17T08:16:27,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for c34c3478acc81ba3f3c8f5caec36c13b: 2024-11-17T08:16:27,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:27,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-17T08:16:27,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-17T08:16:27,820 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-17T08:16:27,820 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2410 sec 2024-11-17T08:16:27,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on c34c3478acc81ba3f3c8f5caec36c13b 2024-11-17T08:16:27,820 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c34c3478acc81ba3f3c8f5caec36c13b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-17T08:16:27,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=A 2024-11-17T08:16:27,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:27,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=B 2024-11-17T08:16:27,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:27,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=C 2024-11-17T08:16:27,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:27,822 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.2460 sec 2024-11-17T08:16:27,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/2799885dff3448af82c85571983bce09 is 50, key is test_row_0/A:col10/1731831387803/Put/seqid=0 2024-11-17T08:16:27,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742293_1469 (size=19321) 2024-11-17T08:16:27,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:27,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831447831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:27,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:27,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831447831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:27,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:27,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:27,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:27,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:27,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831447832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:27,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:27,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:27,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:27,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:27,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:27,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:27,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:27,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33940 deadline: 1731831447839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:27,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:27,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:27,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:27,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:27,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33872 deadline: 1731831447841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:27,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:27,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:27,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:27,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:27,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831447943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:27,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:27,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:27,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831447943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:27,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:27,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:27,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:27,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:27,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:27,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:27,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831447943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:27,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:27,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:27,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:27,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:27,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33940 deadline: 1731831447954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:27,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:27,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:27,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:27,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:27,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33872 deadline: 1731831447964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:27,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:27,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:27,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:28,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:28,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831448153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:28,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:28,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:28,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831448153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:28,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:28,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:28,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:28,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:28,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831448153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:28,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:28,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:28,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:28,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:28,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:28,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:28,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33940 deadline: 1731831448174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:28,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:28,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:28,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:28,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:28,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33872 deadline: 1731831448184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:28,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:28,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:28,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:28,228 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/2799885dff3448af82c85571983bce09 2024-11-17T08:16:28,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/bca29fa91606434d9d4eb6ffd1c277e0 is 50, key is test_row_0/B:col10/1731831387803/Put/seqid=0 2024-11-17T08:16:28,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742294_1470 (size=12151) 2024-11-17T08:16:28,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:28,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831448463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:28,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:28,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:28,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831448463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:28,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:28,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:28,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:28,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:28,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:28,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831448463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:28,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:28,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:28,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:28,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:28,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:28,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33940 deadline: 1731831448494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:28,499 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:28,499 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:28,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:28,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:28,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33872 deadline: 1731831448504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:28,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:28,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:28,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:28,638 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/bca29fa91606434d9d4eb6ffd1c277e0 2024-11-17T08:16:28,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/25180f4dedd5491fa3ebbaeca6fe86d6 is 50, key is test_row_0/C:col10/1731831387803/Put/seqid=0 2024-11-17T08:16:28,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742295_1471 (size=12151) 2024-11-17T08:16:28,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:28,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831448973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:28,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:28,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:28,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:28,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831448974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:28,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:28,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:28,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:28,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:28,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:28,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831448974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:28,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:28,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:28,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:29,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:29,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33940 deadline: 1731831449003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:29,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:29,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:29,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:29,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:29,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33872 deadline: 1731831449023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:29,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:29,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:29,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:29,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/25180f4dedd5491fa3ebbaeca6fe86d6 2024-11-17T08:16:29,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/2799885dff3448af82c85571983bce09 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/2799885dff3448af82c85571983bce09 2024-11-17T08:16:29,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/2799885dff3448af82c85571983bce09, entries=300, sequenceid=149, filesize=18.9 K 2024-11-17T08:16:29,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/bca29fa91606434d9d4eb6ffd1c277e0 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/bca29fa91606434d9d4eb6ffd1c277e0 2024-11-17T08:16:29,058 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/bca29fa91606434d9d4eb6ffd1c277e0, entries=150, sequenceid=149, filesize=11.9 K 2024-11-17T08:16:29,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/25180f4dedd5491fa3ebbaeca6fe86d6 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/25180f4dedd5491fa3ebbaeca6fe86d6 2024-11-17T08:16:29,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/25180f4dedd5491fa3ebbaeca6fe86d6, entries=150, sequenceid=149, filesize=11.9 K 2024-11-17T08:16:29,062 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c34c3478acc81ba3f3c8f5caec36c13b in 1242ms, sequenceid=149, compaction requested=true 2024-11-17T08:16:29,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c34c3478acc81ba3f3c8f5caec36c13b: 2024-11-17T08:16:29,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c34c3478acc81ba3f3c8f5caec36c13b:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:16:29,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:29,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c34c3478acc81ba3f3c8f5caec36c13b:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:16:29,062 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T08:16:29,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:29,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c34c3478acc81ba3f3c8f5caec36c13b:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:16:29,062 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T08:16:29,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:16:29,063 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 60210 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T08:16:29,063 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T08:16:29,063 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): c34c3478acc81ba3f3c8f5caec36c13b/A is initiating minor compaction (all files) 2024-11-17T08:16:29,063 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): c34c3478acc81ba3f3c8f5caec36c13b/B is initiating minor compaction (all files) 2024-11-17T08:16:29,063 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c34c3478acc81ba3f3c8f5caec36c13b/B in TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:29,063 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c34c3478acc81ba3f3c8f5caec36c13b/A in TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:29,063 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/39ef1982fd5841d4ab24daba698d255e, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/e9055076ea234762a747f05593e45962, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/f6e98abedc8c4738abfd3b750295cf01, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/2799885dff3448af82c85571983bce09] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp, totalSize=58.8 K 2024-11-17T08:16:29,063 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/d8612083b1774a49a9ad83a081d79e14, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/47ce2268ce0247eeadaeff11d9f0a9e7, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/30b1e7adf47b457c8d172d7c841b38e0, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/bca29fa91606434d9d4eb6ffd1c277e0] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp, totalSize=47.2 K 2024-11-17T08:16:29,063 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting d8612083b1774a49a9ad83a081d79e14, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731831384372 2024-11-17T08:16:29,063 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39ef1982fd5841d4ab24daba698d255e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731831384372 2024-11-17T08:16:29,063 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9055076ea234762a747f05593e45962, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731831385553 2024-11-17T08:16:29,063 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 47ce2268ce0247eeadaeff11d9f0a9e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731831385574 2024-11-17T08:16:29,064 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6e98abedc8c4738abfd3b750295cf01, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731831385607 2024-11-17T08:16:29,064 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 30b1e7adf47b457c8d172d7c841b38e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731831385607 2024-11-17T08:16:29,064 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2799885dff3448af82c85571983bce09, keycount=300, bloomtype=ROW, size=18.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731831387786 2024-11-17T08:16:29,064 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting bca29fa91606434d9d4eb6ffd1c277e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731831387803 2024-11-17T08:16:29,072 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c34c3478acc81ba3f3c8f5caec36c13b#A#compaction#396 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:29,072 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/f287d38d590b43fd83feada0ccff1ea4 is 50, key is test_row_0/A:col10/1731831387803/Put/seqid=0 2024-11-17T08:16:29,073 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c34c3478acc81ba3f3c8f5caec36c13b#B#compaction#397 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:29,074 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/a429a8019ea746f2bd71f7b9469460c2 is 50, key is test_row_0/B:col10/1731831387803/Put/seqid=0 2024-11-17T08:16:29,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742296_1472 (size=12493) 2024-11-17T08:16:29,084 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/f287d38d590b43fd83feada0ccff1ea4 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/f287d38d590b43fd83feada0ccff1ea4 2024-11-17T08:16:29,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742297_1473 (size=12493) 2024-11-17T08:16:29,089 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c34c3478acc81ba3f3c8f5caec36c13b/A of c34c3478acc81ba3f3c8f5caec36c13b into f287d38d590b43fd83feada0ccff1ea4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:29,089 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c34c3478acc81ba3f3c8f5caec36c13b: 2024-11-17T08:16:29,089 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., storeName=c34c3478acc81ba3f3c8f5caec36c13b/A, priority=12, startTime=1731831389062; duration=0sec 2024-11-17T08:16:29,089 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:16:29,089 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c34c3478acc81ba3f3c8f5caec36c13b:A 2024-11-17T08:16:29,089 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T08:16:29,090 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T08:16:29,090 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): c34c3478acc81ba3f3c8f5caec36c13b/C is initiating minor compaction (all files) 2024-11-17T08:16:29,090 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c34c3478acc81ba3f3c8f5caec36c13b/C in TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:29,091 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/c4b35217441c4ce4a6471c7f8ea176e9, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/de202e4c093c443da3df448adf077de9, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/273f89913f534258ace2097b0e81e67b, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/25180f4dedd5491fa3ebbaeca6fe86d6] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp, totalSize=47.2 K 2024-11-17T08:16:29,091 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4b35217441c4ce4a6471c7f8ea176e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731831384372 2024-11-17T08:16:29,091 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting de202e4c093c443da3df448adf077de9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731831385574 2024-11-17T08:16:29,091 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/a429a8019ea746f2bd71f7b9469460c2 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/a429a8019ea746f2bd71f7b9469460c2 2024-11-17T08:16:29,091 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 273f89913f534258ace2097b0e81e67b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731831385607 2024-11-17T08:16:29,092 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25180f4dedd5491fa3ebbaeca6fe86d6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731831387803 2024-11-17T08:16:29,095 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c34c3478acc81ba3f3c8f5caec36c13b/B of c34c3478acc81ba3f3c8f5caec36c13b into a429a8019ea746f2bd71f7b9469460c2(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:29,095 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c34c3478acc81ba3f3c8f5caec36c13b: 2024-11-17T08:16:29,095 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., storeName=c34c3478acc81ba3f3c8f5caec36c13b/B, priority=12, startTime=1731831389062; duration=0sec 2024-11-17T08:16:29,095 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:29,095 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c34c3478acc81ba3f3c8f5caec36c13b:B 2024-11-17T08:16:29,100 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c34c3478acc81ba3f3c8f5caec36c13b#C#compaction#398 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:29,100 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/6b1c65eeb8ab47c08ce994fb0c40a74b is 50, key is test_row_0/C:col10/1731831387803/Put/seqid=0 2024-11-17T08:16:29,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742298_1474 (size=12493) 2024-11-17T08:16:29,507 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/6b1c65eeb8ab47c08ce994fb0c40a74b as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/6b1c65eeb8ab47c08ce994fb0c40a74b 2024-11-17T08:16:29,510 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c34c3478acc81ba3f3c8f5caec36c13b/C of c34c3478acc81ba3f3c8f5caec36c13b into 6b1c65eeb8ab47c08ce994fb0c40a74b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:29,510 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c34c3478acc81ba3f3c8f5caec36c13b: 2024-11-17T08:16:29,510 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., storeName=c34c3478acc81ba3f3c8f5caec36c13b/C, priority=12, startTime=1731831389062; duration=0sec 2024-11-17T08:16:29,511 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:29,511 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c34c3478acc81ba3f3c8f5caec36c13b:C 2024-11-17T08:16:29,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T08:16:29,744 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:16:29,744 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:16:29,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-17T08:16:29,745 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:16:29,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T08:16:29,746 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:16:29,746 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:16:29,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T08:16:29,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-17T08:16:29,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:29,898 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing c34c3478acc81ba3f3c8f5caec36c13b 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-17T08:16:29,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=A 2024-11-17T08:16:29,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:29,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=B 2024-11-17T08:16:29,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:29,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=C 2024-11-17T08:16:29,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:29,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/c89d961a628f424caf1b3e8b01bd2185 is 50, key is test_row_0/A:col10/1731831387838/Put/seqid=0 2024-11-17T08:16:29,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742299_1475 (size=12151) 2024-11-17T08:16:29,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on c34c3478acc81ba3f3c8f5caec36c13b 2024-11-17T08:16:29,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. as already flushing 2024-11-17T08:16:30,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:30,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831450029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:30,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:30,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:30,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:30,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:30,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33940 deadline: 1731831450030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:30,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:30,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:30,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:30,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831450031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:30,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:30,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:30,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:30,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:30,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:30,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831450036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:30,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:30,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:30,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:30,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:30,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33872 deadline: 1731831450043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:30,052 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to e3d0adb87c01,40631,1731831247805 for 'test_row_2' in c34c3478acc81ba3f3c8f5caec36c13b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T08:16:30,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:30,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:30,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:30,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T08:16:30,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:30,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831450144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:30,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:30,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33940 deadline: 1731831450144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:30,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:30,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:30,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831450144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:30,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:30,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831450144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:30,305 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/c89d961a628f424caf1b3e8b01bd2185 2024-11-17T08:16:30,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/31f7710a86bd4949a44328e9e67485e0 is 50, key is test_row_0/B:col10/1731831387838/Put/seqid=0 2024-11-17T08:16:30,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742300_1476 (size=12151) 2024-11-17T08:16:30,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:30,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33940 deadline: 1731831450353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:30,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:30,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831450353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:30,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:30,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:30,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:30,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:30,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:30,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:30,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:30,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831450363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:30,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:30,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:30,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:30,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831450363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:30,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:30,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:30,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:30,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:30,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T08:16:30,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:30,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33940 deadline: 1731831450664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:30,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:30,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831450664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:30,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:30,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:30,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:30,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:30,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:30,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:30,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:30,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831450673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:30,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:30,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:30,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:30,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831450673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:30,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:30,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:30,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:30,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:30,714 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/31f7710a86bd4949a44328e9e67485e0 2024-11-17T08:16:30,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/4ba557d762aa47ed8e84e0567616cce8 is 50, key is test_row_0/C:col10/1731831387838/Put/seqid=0 2024-11-17T08:16:30,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742301_1477 (size=12151) 2024-11-17T08:16:30,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T08:16:31,123 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/4ba557d762aa47ed8e84e0567616cce8 2024-11-17T08:16:31,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/c89d961a628f424caf1b3e8b01bd2185 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/c89d961a628f424caf1b3e8b01bd2185 2024-11-17T08:16:31,130 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/c89d961a628f424caf1b3e8b01bd2185, entries=150, sequenceid=164, filesize=11.9 K 2024-11-17T08:16:31,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/31f7710a86bd4949a44328e9e67485e0 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/31f7710a86bd4949a44328e9e67485e0 2024-11-17T08:16:31,133 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/31f7710a86bd4949a44328e9e67485e0, entries=150, sequenceid=164, filesize=11.9 K 2024-11-17T08:16:31,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/4ba557d762aa47ed8e84e0567616cce8 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/4ba557d762aa47ed8e84e0567616cce8 2024-11-17T08:16:31,136 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/4ba557d762aa47ed8e84e0567616cce8, entries=150, sequenceid=164, filesize=11.9 K 2024-11-17T08:16:31,137 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c34c3478acc81ba3f3c8f5caec36c13b in 1239ms, sequenceid=164, compaction requested=false 2024-11-17T08:16:31,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for c34c3478acc81ba3f3c8f5caec36c13b: 2024-11-17T08:16:31,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:31,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-17T08:16:31,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-17T08:16:31,139 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-17T08:16:31,139 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3920 sec 2024-11-17T08:16:31,141 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.3960 sec 2024-11-17T08:16:31,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on c34c3478acc81ba3f3c8f5caec36c13b 2024-11-17T08:16:31,190 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c34c3478acc81ba3f3c8f5caec36c13b 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-17T08:16:31,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=A 2024-11-17T08:16:31,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:16:31,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=B 2024-11-17T08:16:31,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:16:31,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=C 2024-11-17T08:16:31,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:16:31,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/871b560ff2d642a7a2f227fafbe8ccdf is 50, key is test_row_0/A:col10/1731831390019/Put/seqid=0 2024-11-17T08:16:31,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742302_1478 (size=14541) 2024-11-17T08:16:31,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:31,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831451203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:31,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:31,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:31,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:31,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:31,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831451203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:31,212 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:31,212 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:31,212 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:31,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:31,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33940 deadline: 1731831451204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:31,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:31,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:31,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:31,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:31,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831451205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:31,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:31,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:31,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:31,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:31,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831451313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:31,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:31,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:31,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831451313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:31,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:31,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:31,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:31,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:31,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:31,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:31,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33940 deadline: 1731831451323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:31,333 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:31,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:31,333 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:31,333 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:31,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831451324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:31,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:31,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:31,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:31,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:31,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831451523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:31,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:31,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:31,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:31,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:31,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831451533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:31,542 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:31,542 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:31,542 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:31,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33940 deadline: 1731831451544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:31,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:31,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831451544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:31,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:31,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:31,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:31,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:31,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:31,602 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/871b560ff2d642a7a2f227fafbe8ccdf 2024-11-17T08:16:31,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/e38b70bf1c3b4e81a9a008a18292c32f is 50, key is test_row_0/B:col10/1731831390019/Put/seqid=0 2024-11-17T08:16:31,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742303_1479 (size=12151) 2024-11-17T08:16:31,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:31,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831451834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:31,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:31,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:31,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:31,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:31,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831451853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:31,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:31,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:31,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33940 deadline: 1731831451853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:31,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:31,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:31,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:31,861 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:31,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831451853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:31,861 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:31,861 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:31,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:31,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:31,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:31,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T08:16:31,894 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:16:31,894 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:16:31,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-17T08:16:31,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T08:16:31,896 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:16:31,896 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:16:31,896 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:16:32,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T08:16:32,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/e38b70bf1c3b4e81a9a008a18292c32f 2024-11-17T08:16:32,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/7b08bddd166f42c0957b727aaab410e7 is 50, key is test_row_0/C:col10/1731831390019/Put/seqid=0 2024-11-17T08:16:32,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742304_1480 (size=12151) 2024-11-17T08:16:32,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T08:16:32,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:32,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. as already flushing 2024-11-17T08:16:32,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:32,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:32,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:32,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:32,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:32,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33872 deadline: 1731831452073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:32,082 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to e3d0adb87c01,40631,1731831247805 for 'test_row_2' in c34c3478acc81ba3f3c8f5caec36c13b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T08:16:32,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:32,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:32,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:32,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T08:16:32,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:32,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. as already flushing 2024-11-17T08:16:32,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:32,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:32,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:32,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:32,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T08:16:32,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T08:16:32,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:32,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. as already flushing 2024-11-17T08:16:32,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:32,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:32,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:32,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:32,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:32,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831452353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:32,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:32,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:32,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:32,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:32,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831452363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:32,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:32,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:32,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:32,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33940 deadline: 1731831452363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:32,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:32,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:32,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:32,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:32,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:32,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831452374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:32,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/7b08bddd166f42c0957b727aaab410e7 2024-11-17T08:16:32,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/871b560ff2d642a7a2f227fafbe8ccdf as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/871b560ff2d642a7a2f227fafbe8ccdf 2024-11-17T08:16:32,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/871b560ff2d642a7a2f227fafbe8ccdf, entries=200, sequenceid=185, filesize=14.2 K 2024-11-17T08:16:32,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/e38b70bf1c3b4e81a9a008a18292c32f as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/e38b70bf1c3b4e81a9a008a18292c32f 2024-11-17T08:16:32,431 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/e38b70bf1c3b4e81a9a008a18292c32f, entries=150, sequenceid=185, filesize=11.9 K 2024-11-17T08:16:32,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/7b08bddd166f42c0957b727aaab410e7 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/7b08bddd166f42c0957b727aaab410e7 2024-11-17T08:16:32,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/7b08bddd166f42c0957b727aaab410e7, entries=150, sequenceid=185, filesize=11.9 K 2024-11-17T08:16:32,435 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for c34c3478acc81ba3f3c8f5caec36c13b in 1246ms, sequenceid=185, compaction requested=true 2024-11-17T08:16:32,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c34c3478acc81ba3f3c8f5caec36c13b: 2024-11-17T08:16:32,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c34c3478acc81ba3f3c8f5caec36c13b:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:16:32,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:32,435 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:16:32,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c34c3478acc81ba3f3c8f5caec36c13b:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:16:32,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:16:32,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c34c3478acc81ba3f3c8f5caec36c13b:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:16:32,435 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:16:32,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:16:32,437 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:16:32,437 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:16:32,437 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): c34c3478acc81ba3f3c8f5caec36c13b/A is initiating minor compaction (all files) 2024-11-17T08:16:32,437 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): c34c3478acc81ba3f3c8f5caec36c13b/B is initiating minor compaction (all files) 2024-11-17T08:16:32,437 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c34c3478acc81ba3f3c8f5caec36c13b/A in TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:32,437 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c34c3478acc81ba3f3c8f5caec36c13b/B in TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:32,437 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/f287d38d590b43fd83feada0ccff1ea4, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/c89d961a628f424caf1b3e8b01bd2185, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/871b560ff2d642a7a2f227fafbe8ccdf] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp, totalSize=38.3 K 2024-11-17T08:16:32,437 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/a429a8019ea746f2bd71f7b9469460c2, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/31f7710a86bd4949a44328e9e67485e0, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/e38b70bf1c3b4e81a9a008a18292c32f] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp, totalSize=35.9 K 2024-11-17T08:16:32,437 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting a429a8019ea746f2bd71f7b9469460c2, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731831387803 2024-11-17T08:16:32,437 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting f287d38d590b43fd83feada0ccff1ea4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731831387803 2024-11-17T08:16:32,437 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting c89d961a628f424caf1b3e8b01bd2185, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731831387830 2024-11-17T08:16:32,437 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 31f7710a86bd4949a44328e9e67485e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731831387830 2024-11-17T08:16:32,438 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 871b560ff2d642a7a2f227fafbe8ccdf, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731831390019 2024-11-17T08:16:32,438 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting e38b70bf1c3b4e81a9a008a18292c32f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731831390019 2024-11-17T08:16:32,443 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c34c3478acc81ba3f3c8f5caec36c13b#A#compaction#405 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:32,443 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/03a31717b0ae456692787c0b3ebde4c6 is 50, key is test_row_0/A:col10/1731831390019/Put/seqid=0 2024-11-17T08:16:32,443 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c34c3478acc81ba3f3c8f5caec36c13b#B#compaction#406 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:32,444 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/489d211e73ef4117b402648a7d4dfe29 is 50, key is test_row_0/B:col10/1731831390019/Put/seqid=0 2024-11-17T08:16:32,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742305_1481 (size=12595) 2024-11-17T08:16:32,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742306_1482 (size=12595) 2024-11-17T08:16:32,452 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/489d211e73ef4117b402648a7d4dfe29 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/489d211e73ef4117b402648a7d4dfe29 2024-11-17T08:16:32,455 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c34c3478acc81ba3f3c8f5caec36c13b/B of c34c3478acc81ba3f3c8f5caec36c13b into 489d211e73ef4117b402648a7d4dfe29(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:32,455 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c34c3478acc81ba3f3c8f5caec36c13b: 2024-11-17T08:16:32,455 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., storeName=c34c3478acc81ba3f3c8f5caec36c13b/B, priority=13, startTime=1731831392435; duration=0sec 2024-11-17T08:16:32,455 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:16:32,455 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c34c3478acc81ba3f3c8f5caec36c13b:B 2024-11-17T08:16:32,455 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:16:32,456 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:16:32,456 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): c34c3478acc81ba3f3c8f5caec36c13b/C is initiating minor compaction (all files) 2024-11-17T08:16:32,456 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c34c3478acc81ba3f3c8f5caec36c13b/C in TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:32,456 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/6b1c65eeb8ab47c08ce994fb0c40a74b, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/4ba557d762aa47ed8e84e0567616cce8, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/7b08bddd166f42c0957b727aaab410e7] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp, totalSize=35.9 K 2024-11-17T08:16:32,456 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b1c65eeb8ab47c08ce994fb0c40a74b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731831387803 2024-11-17T08:16:32,457 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ba557d762aa47ed8e84e0567616cce8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731831387830 2024-11-17T08:16:32,457 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b08bddd166f42c0957b727aaab410e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731831390019 2024-11-17T08:16:32,462 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c34c3478acc81ba3f3c8f5caec36c13b#C#compaction#407 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:32,462 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/4abc988953c948a08319e4515f6dd030 is 50, key is test_row_0/C:col10/1731831390019/Put/seqid=0 2024-11-17T08:16:32,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742307_1483 (size=12595) 2024-11-17T08:16:32,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T08:16:32,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:32,506 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing c34c3478acc81ba3f3c8f5caec36c13b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-17T08:16:32,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=A 2024-11-17T08:16:32,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:32,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=B 2024-11-17T08:16:32,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:32,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=C 2024-11-17T08:16:32,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:32,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/d614ca6ba8c94756b71e769130774810 is 50, key is test_row_0/A:col10/1731831391203/Put/seqid=0 2024-11-17T08:16:32,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742308_1484 (size=12151) 2024-11-17T08:16:32,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T08:16:32,853 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/03a31717b0ae456692787c0b3ebde4c6 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/03a31717b0ae456692787c0b3ebde4c6 2024-11-17T08:16:32,856 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c34c3478acc81ba3f3c8f5caec36c13b/A of c34c3478acc81ba3f3c8f5caec36c13b into 03a31717b0ae456692787c0b3ebde4c6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:32,856 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c34c3478acc81ba3f3c8f5caec36c13b: 2024-11-17T08:16:32,856 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., storeName=c34c3478acc81ba3f3c8f5caec36c13b/A, priority=13, startTime=1731831392435; duration=0sec 2024-11-17T08:16:32,856 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:32,856 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c34c3478acc81ba3f3c8f5caec36c13b:A 2024-11-17T08:16:32,870 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/4abc988953c948a08319e4515f6dd030 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/4abc988953c948a08319e4515f6dd030 2024-11-17T08:16:32,873 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c34c3478acc81ba3f3c8f5caec36c13b/C of c34c3478acc81ba3f3c8f5caec36c13b into 4abc988953c948a08319e4515f6dd030(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:32,873 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c34c3478acc81ba3f3c8f5caec36c13b: 2024-11-17T08:16:32,873 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., storeName=c34c3478acc81ba3f3c8f5caec36c13b/C, priority=13, startTime=1731831392435; duration=0sec 2024-11-17T08:16:32,873 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:32,873 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c34c3478acc81ba3f3c8f5caec36c13b:C 2024-11-17T08:16:32,913 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/d614ca6ba8c94756b71e769130774810 2024-11-17T08:16:32,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/4e8e7f28b4394274b68ff3b63072ff2c is 50, key is test_row_0/B:col10/1731831391203/Put/seqid=0 2024-11-17T08:16:32,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742309_1485 (size=12151) 2024-11-17T08:16:33,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T08:16:33,325 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/4e8e7f28b4394274b68ff3b63072ff2c 2024-11-17T08:16:33,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/7b423f312a864dc1b0edd856138b36de is 50, key is test_row_0/C:col10/1731831391203/Put/seqid=0 2024-11-17T08:16:33,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742310_1486 (size=12151) 2024-11-17T08:16:33,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on c34c3478acc81ba3f3c8f5caec36c13b 2024-11-17T08:16:33,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. as already flushing 2024-11-17T08:16:33,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:33,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831453409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:33,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:33,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33940 deadline: 1731831453412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:33,421 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731831393404, max timestamp=1731831393418Num uniques 859; 2024-11-17T08:16:33,421 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731831393404, max timestamp=1731831393418Num uniques 859; 2024-11-17T08:16:33,421 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T08:16:33,421 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T08:16:33,421 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731831393404, max timestamp=1731831393418Num uniques 859; 2024-11-17T08:16:33,421 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T08:16:33,421 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:33,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:33,421 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:33,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:33,421 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:33,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:33,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:33,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831453475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:33,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:33,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:33,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:33,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:33,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831453476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:33,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:33,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:33,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:33,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:33,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33940 deadline: 1731831453523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:33,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:33,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:33,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831453523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:33,529 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:33,529 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:33,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:33,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:33,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:33,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:33,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831453584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:33,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:33,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:33,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:33,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:33,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831453593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:33,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:33,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:33,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:33,733 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/7b423f312a864dc1b0edd856138b36de 2024-11-17T08:16:33,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/d614ca6ba8c94756b71e769130774810 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/d614ca6ba8c94756b71e769130774810 2024-11-17T08:16:33,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:33,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33940 deadline: 1731831453733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:33,744 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/d614ca6ba8c94756b71e769130774810, entries=150, sequenceid=197, filesize=11.9 K 2024-11-17T08:16:33,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:33,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831453734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:33,745 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:33,745 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:33,745 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:33,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:33,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:33,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:33,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/4e8e7f28b4394274b68ff3b63072ff2c as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/4e8e7f28b4394274b68ff3b63072ff2c 2024-11-17T08:16:33,748 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/4e8e7f28b4394274b68ff3b63072ff2c, entries=150, sequenceid=197, filesize=11.9 K 2024-11-17T08:16:33,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/7b423f312a864dc1b0edd856138b36de as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/7b423f312a864dc1b0edd856138b36de 2024-11-17T08:16:33,752 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/7b423f312a864dc1b0edd856138b36de, entries=150, sequenceid=197, filesize=11.9 K 2024-11-17T08:16:33,753 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for c34c3478acc81ba3f3c8f5caec36c13b in 1247ms, sequenceid=197, compaction requested=false 2024-11-17T08:16:33,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for c34c3478acc81ba3f3c8f5caec36c13b: 2024-11-17T08:16:33,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:33,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-17T08:16:33,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-17T08:16:33,757 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-17T08:16:33,757 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8590 sec 2024-11-17T08:16:33,758 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.8630 sec 2024-11-17T08:16:33,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on c34c3478acc81ba3f3c8f5caec36c13b 2024-11-17T08:16:33,804 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c34c3478acc81ba3f3c8f5caec36c13b 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-17T08:16:33,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=A 2024-11-17T08:16:33,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:16:33,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=B 2024-11-17T08:16:33,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:16:33,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=C 2024-11-17T08:16:33,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:16:33,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/7ee73e82d4d445a6b7835d6a6a832d5a is 50, key is test_row_0/A:col10/1731831393474/Put/seqid=0 2024-11-17T08:16:33,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742311_1487 (size=14541) 2024-11-17T08:16:33,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:33,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831453830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:33,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:33,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:33,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831453831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:33,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:33,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:33,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:33,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:33,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:33,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:33,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831453944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:33,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:33,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:33,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:33,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831453944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:33,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:33,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:33,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:33,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:34,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T08:16:34,054 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:16:34,054 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:16:34,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-17T08:16:34,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T08:16:34,056 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:16:34,056 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:16:34,057 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:16:34,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:34,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33940 deadline: 1731831454054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:34,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:34,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:34,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831454054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:34,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:34,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:34,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:34,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:34,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:34,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:34,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831454154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:34,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:34,161 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:34,161 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:34,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831454154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:34,161 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:34,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:34,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:34,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:34,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T08:16:34,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T08:16:34,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:34,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. as already flushing 2024-11-17T08:16:34,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:34,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:34,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:34,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:34,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/7ee73e82d4d445a6b7835d6a6a832d5a 2024-11-17T08:16:34,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/73d3437d1cd1416da35ba54ba7419f5d is 50, key is test_row_0/B:col10/1731831393474/Put/seqid=0 2024-11-17T08:16:34,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742312_1488 (size=12151) 2024-11-17T08:16:34,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/73d3437d1cd1416da35ba54ba7419f5d 2024-11-17T08:16:34,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/7d98e73a50ec450d959090288ac58c04 is 50, key is test_row_0/C:col10/1731831393474/Put/seqid=0 2024-11-17T08:16:34,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742313_1489 (size=12151) 2024-11-17T08:16:34,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T08:16:34,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:34,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. as already flushing 2024-11-17T08:16:34,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:34,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:34,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:34,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:34,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T08:16:34,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:34,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831454463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:34,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:34,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831454473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:34,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:34,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:34,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:34,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T08:16:34,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:34,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. as already flushing 2024-11-17T08:16:34,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:34,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:34,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:34,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:34,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:34,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33940 deadline: 1731831454573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:34,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:34,575 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:34,575 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:34,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831454573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:34,575 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:34,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:34,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:34,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:34,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/7d98e73a50ec450d959090288ac58c04 2024-11-17T08:16:34,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/7ee73e82d4d445a6b7835d6a6a832d5a as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/7ee73e82d4d445a6b7835d6a6a832d5a 2024-11-17T08:16:34,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/7ee73e82d4d445a6b7835d6a6a832d5a, entries=200, sequenceid=236, filesize=14.2 K 2024-11-17T08:16:34,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/73d3437d1cd1416da35ba54ba7419f5d as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/73d3437d1cd1416da35ba54ba7419f5d 2024-11-17T08:16:34,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/73d3437d1cd1416da35ba54ba7419f5d, entries=150, sequenceid=236, filesize=11.9 K 2024-11-17T08:16:34,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/7d98e73a50ec450d959090288ac58c04 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/7d98e73a50ec450d959090288ac58c04 2024-11-17T08:16:34,664 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/7d98e73a50ec450d959090288ac58c04, entries=150, sequenceid=236, filesize=11.9 K 2024-11-17T08:16:34,664 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for c34c3478acc81ba3f3c8f5caec36c13b in 860ms, sequenceid=236, compaction requested=true 2024-11-17T08:16:34,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c34c3478acc81ba3f3c8f5caec36c13b: 2024-11-17T08:16:34,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c34c3478acc81ba3f3c8f5caec36c13b:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:16:34,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:34,664 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:16:34,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c34c3478acc81ba3f3c8f5caec36c13b:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:16:34,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:34,664 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:16:34,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c34c3478acc81ba3f3c8f5caec36c13b:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:16:34,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:16:34,665 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:16:34,665 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:16:34,665 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): c34c3478acc81ba3f3c8f5caec36c13b/B is initiating minor compaction (all files) 2024-11-17T08:16:34,665 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): c34c3478acc81ba3f3c8f5caec36c13b/A is initiating minor compaction (all files) 2024-11-17T08:16:34,665 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c34c3478acc81ba3f3c8f5caec36c13b/B in TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:34,665 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c34c3478acc81ba3f3c8f5caec36c13b/A in TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:34,665 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/489d211e73ef4117b402648a7d4dfe29, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/4e8e7f28b4394274b68ff3b63072ff2c, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/73d3437d1cd1416da35ba54ba7419f5d] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp, totalSize=36.0 K 2024-11-17T08:16:34,665 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/03a31717b0ae456692787c0b3ebde4c6, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/d614ca6ba8c94756b71e769130774810, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/7ee73e82d4d445a6b7835d6a6a832d5a] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp, totalSize=38.4 K 2024-11-17T08:16:34,666 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 489d211e73ef4117b402648a7d4dfe29, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731831390019 2024-11-17T08:16:34,666 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03a31717b0ae456692787c0b3ebde4c6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731831390019 2024-11-17T08:16:34,667 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e8e7f28b4394274b68ff3b63072ff2c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731831391202 2024-11-17T08:16:34,667 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting d614ca6ba8c94756b71e769130774810, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731831391202 2024-11-17T08:16:34,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T08:16:34,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:34,667 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing c34c3478acc81ba3f3c8f5caec36c13b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T08:16:34,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=A 2024-11-17T08:16:34,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:34,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=B 2024-11-17T08:16:34,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:34,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=C 2024-11-17T08:16:34,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:34,669 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 73d3437d1cd1416da35ba54ba7419f5d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731831393474 2024-11-17T08:16:34,669 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ee73e82d4d445a6b7835d6a6a832d5a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731831393470 2024-11-17T08:16:34,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/c1743033d0e9407091109eef43397391 is 50, key is test_row_0/A:col10/1731831393830/Put/seqid=0 2024-11-17T08:16:34,674 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c34c3478acc81ba3f3c8f5caec36c13b#B#compaction#416 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:34,674 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c34c3478acc81ba3f3c8f5caec36c13b#A#compaction#415 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:34,675 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/5d3b66f196384e1fa6761009fe2f4d97 is 50, key is test_row_0/A:col10/1731831393474/Put/seqid=0 2024-11-17T08:16:34,675 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/2cccc733678e493abd4ebee27f713476 is 50, key is test_row_0/B:col10/1731831393474/Put/seqid=0 2024-11-17T08:16:34,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742314_1490 (size=9757) 2024-11-17T08:16:34,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742315_1491 (size=12697) 2024-11-17T08:16:34,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742316_1492 (size=12697) 2024-11-17T08:16:34,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T08:16:34,687 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/5d3b66f196384e1fa6761009fe2f4d97 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/5d3b66f196384e1fa6761009fe2f4d97 2024-11-17T08:16:34,693 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c34c3478acc81ba3f3c8f5caec36c13b/A of c34c3478acc81ba3f3c8f5caec36c13b into 5d3b66f196384e1fa6761009fe2f4d97(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:34,693 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c34c3478acc81ba3f3c8f5caec36c13b: 2024-11-17T08:16:34,693 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., storeName=c34c3478acc81ba3f3c8f5caec36c13b/A, priority=13, startTime=1731831394664; duration=0sec 2024-11-17T08:16:34,693 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:16:34,693 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c34c3478acc81ba3f3c8f5caec36c13b:A 2024-11-17T08:16:34,693 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:16:34,694 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:16:34,694 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): c34c3478acc81ba3f3c8f5caec36c13b/C is initiating minor compaction (all files) 2024-11-17T08:16:34,694 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c34c3478acc81ba3f3c8f5caec36c13b/C in TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:34,694 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/4abc988953c948a08319e4515f6dd030, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/7b423f312a864dc1b0edd856138b36de, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/7d98e73a50ec450d959090288ac58c04] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp, totalSize=36.0 K 2024-11-17T08:16:34,694 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4abc988953c948a08319e4515f6dd030, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731831390019 2024-11-17T08:16:34,694 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b423f312a864dc1b0edd856138b36de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731831391202 2024-11-17T08:16:34,695 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d98e73a50ec450d959090288ac58c04, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731831393474 2024-11-17T08:16:34,701 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c34c3478acc81ba3f3c8f5caec36c13b#C#compaction#417 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:34,701 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/ae7294bb9b8543f0899ee1a3c2080be4 is 50, key is test_row_0/C:col10/1731831393474/Put/seqid=0 2024-11-17T08:16:34,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742317_1493 (size=12697) 2024-11-17T08:16:34,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on c34c3478acc81ba3f3c8f5caec36c13b 2024-11-17T08:16:34,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. as already flushing 2024-11-17T08:16:35,037 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731831395017, max timestamp=1731831395034Num uniques 859; 2024-11-17T08:16:35,037 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T08:16:35,037 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731831395017, max timestamp=1731831395034Num uniques 859; 2024-11-17T08:16:35,037 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T08:16:35,037 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731831395017, max timestamp=1731831395034Num uniques 859; 2024-11-17T08:16:35,037 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T08:16:35,077 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/c1743033d0e9407091109eef43397391 2024-11-17T08:16:35,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/28bbbbda17a44465937c6a36852c27eb is 50, key is test_row_0/B:col10/1731831393830/Put/seqid=0 2024-11-17T08:16:35,084 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/2cccc733678e493abd4ebee27f713476 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/2cccc733678e493abd4ebee27f713476 2024-11-17T08:16:35,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742318_1494 (size=9757) 2024-11-17T08:16:35,088 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c34c3478acc81ba3f3c8f5caec36c13b/B of c34c3478acc81ba3f3c8f5caec36c13b into 2cccc733678e493abd4ebee27f713476(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:35,088 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c34c3478acc81ba3f3c8f5caec36c13b: 2024-11-17T08:16:35,088 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., storeName=c34c3478acc81ba3f3c8f5caec36c13b/B, priority=13, startTime=1731831394664; duration=0sec 2024-11-17T08:16:35,088 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:35,088 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c34c3478acc81ba3f3c8f5caec36c13b:B 2024-11-17T08:16:35,110 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/ae7294bb9b8543f0899ee1a3c2080be4 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/ae7294bb9b8543f0899ee1a3c2080be4 2024-11-17T08:16:35,113 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c34c3478acc81ba3f3c8f5caec36c13b/C of c34c3478acc81ba3f3c8f5caec36c13b into ae7294bb9b8543f0899ee1a3c2080be4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:35,113 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c34c3478acc81ba3f3c8f5caec36c13b: 2024-11-17T08:16:35,113 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., storeName=c34c3478acc81ba3f3c8f5caec36c13b/C, priority=13, startTime=1731831394664; duration=0sec 2024-11-17T08:16:35,113 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:35,113 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c34c3478acc81ba3f3c8f5caec36c13b:C 2024-11-17T08:16:35,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:35,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831455121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:35,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:35,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:35,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:35,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:35,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831455124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:35,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:35,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:35,137 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:35,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T08:16:35,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:35,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831455243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:35,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:35,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:35,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:35,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831455243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:35,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:35,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:35,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:35,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:35,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:35,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831455454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:35,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:35,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:35,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831455454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:35,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:35,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:35,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:35,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:35,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:35,487 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/28bbbbda17a44465937c6a36852c27eb 2024-11-17T08:16:35,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/9c7ec94cf2e244b3b928fd9fef0d7b17 is 50, key is test_row_0/C:col10/1731831393830/Put/seqid=0 2024-11-17T08:16:35,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742319_1495 (size=9757) 2024-11-17T08:16:35,498 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/9c7ec94cf2e244b3b928fd9fef0d7b17 2024-11-17T08:16:35,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/c1743033d0e9407091109eef43397391 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/c1743033d0e9407091109eef43397391 2024-11-17T08:16:35,504 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/c1743033d0e9407091109eef43397391, entries=100, sequenceid=246, filesize=9.5 K 2024-11-17T08:16:35,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/28bbbbda17a44465937c6a36852c27eb as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/28bbbbda17a44465937c6a36852c27eb 2024-11-17T08:16:35,508 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/28bbbbda17a44465937c6a36852c27eb, entries=100, sequenceid=246, filesize=9.5 K 2024-11-17T08:16:35,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/9c7ec94cf2e244b3b928fd9fef0d7b17 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/9c7ec94cf2e244b3b928fd9fef0d7b17 2024-11-17T08:16:35,512 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/9c7ec94cf2e244b3b928fd9fef0d7b17, entries=100, sequenceid=246, filesize=9.5 K 2024-11-17T08:16:35,513 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for c34c3478acc81ba3f3c8f5caec36c13b in 846ms, sequenceid=246, compaction requested=false 2024-11-17T08:16:35,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for c34c3478acc81ba3f3c8f5caec36c13b: 2024-11-17T08:16:35,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:35,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-17T08:16:35,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-17T08:16:35,516 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-17T08:16:35,516 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4570 sec 2024-11-17T08:16:35,517 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.4620 sec 2024-11-17T08:16:35,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on c34c3478acc81ba3f3c8f5caec36c13b 2024-11-17T08:16:35,589 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c34c3478acc81ba3f3c8f5caec36c13b 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-17T08:16:35,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=A 2024-11-17T08:16:35,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:16:35,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=B 2024-11-17T08:16:35,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:16:35,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=C 2024-11-17T08:16:35,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:16:35,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/eb2e64488e634039a583af91adadf73f is 50, key is test_row_0/A:col10/1731831395099/Put/seqid=0 2024-11-17T08:16:35,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742320_1496 (size=14741) 2024-11-17T08:16:35,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:35,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831455607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:35,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:35,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:35,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:35,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:35,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33940 deadline: 1731831455613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:35,618 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:35,618 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:35,618 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:35,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:35,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831455724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:35,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:35,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:35,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33940 deadline: 1731831455724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:35,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:35,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:35,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:35,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:35,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:35,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:35,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831455773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:35,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:35,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:35,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:35,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:35,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831455774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:35,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:35,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:35,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:35,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:35,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831455934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:35,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:35,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:35,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:35,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33940 deadline: 1731831455934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:35,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:35,940 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:35,940 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:35,940 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:35,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/eb2e64488e634039a583af91adadf73f 2024-11-17T08:16:36,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/fa66fdcec4874397b07e839f1037ecca is 50, key is test_row_0/B:col10/1731831395099/Put/seqid=0 2024-11-17T08:16:36,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742321_1497 (size=12301) 2024-11-17T08:16:36,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:36,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33872 deadline: 1731831456103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:36,111 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to e3d0adb87c01,40631,1731831247805 for 'test_row_2' in c34c3478acc81ba3f3c8f5caec36c13b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8270 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T08:16:36,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:36,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:36,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:36,111 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T08:16:36,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T08:16:36,214 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:16:36,215 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:16:36,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-17T08:16:36,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T08:16:36,217 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:16:36,217 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:16:36,218 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:16:36,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:36,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831456244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:36,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:36,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:36,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33940 deadline: 1731831456244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:36,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:36,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:36,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:36,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:36,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:36,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:36,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831456293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:36,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:36,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:36,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831456293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:36,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:36,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:36,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:36,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:36,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:36,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T08:16:36,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-17T08:16:36,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:36,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. as already flushing 2024-11-17T08:16:36,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:36,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:36,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:36,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:36,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/fa66fdcec4874397b07e839f1037ecca 2024-11-17T08:16:36,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/a2f117896f914daf8949f3286b57d3a6 is 50, key is test_row_0/C:col10/1731831395099/Put/seqid=0 2024-11-17T08:16:36,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742322_1498 (size=12301) 2024-11-17T08:16:36,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-17T08:16:36,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:36,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. as already flushing 2024-11-17T08:16:36,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:36,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:36,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:36,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:36,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T08:16:36,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-17T08:16:36,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:36,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. as already flushing 2024-11-17T08:16:36,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:36,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:36,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:36,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:36,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:36,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831456764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:36,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:36,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:36,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33940 deadline: 1731831456764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:36,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:36,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:36,770 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:36,770 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:36,770 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:36,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-17T08:16:36,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:36,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. as already flushing 2024-11-17T08:16:36,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:36,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:36,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:36,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:36,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T08:16:36,860 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/a2f117896f914daf8949f3286b57d3a6 2024-11-17T08:16:36,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/eb2e64488e634039a583af91adadf73f as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/eb2e64488e634039a583af91adadf73f 2024-11-17T08:16:36,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/eb2e64488e634039a583af91adadf73f, entries=200, sequenceid=286, filesize=14.4 K 2024-11-17T08:16:36,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/fa66fdcec4874397b07e839f1037ecca as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/fa66fdcec4874397b07e839f1037ecca 2024-11-17T08:16:36,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/fa66fdcec4874397b07e839f1037ecca, entries=150, sequenceid=286, filesize=12.0 K 2024-11-17T08:16:36,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/a2f117896f914daf8949f3286b57d3a6 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/a2f117896f914daf8949f3286b57d3a6 2024-11-17T08:16:36,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/a2f117896f914daf8949f3286b57d3a6, entries=150, sequenceid=286, filesize=12.0 K 2024-11-17T08:16:36,873 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for c34c3478acc81ba3f3c8f5caec36c13b in 1284ms, sequenceid=286, compaction requested=true 2024-11-17T08:16:36,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c34c3478acc81ba3f3c8f5caec36c13b: 2024-11-17T08:16:36,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c34c3478acc81ba3f3c8f5caec36c13b:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:16:36,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:36,874 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:16:36,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c34c3478acc81ba3f3c8f5caec36c13b:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:16:36,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:36,874 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:16:36,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c34c3478acc81ba3f3c8f5caec36c13b:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:16:36,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:16:36,874 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:16:36,874 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:16:36,874 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): c34c3478acc81ba3f3c8f5caec36c13b/A is initiating minor compaction (all files) 2024-11-17T08:16:36,874 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): c34c3478acc81ba3f3c8f5caec36c13b/B is initiating minor compaction (all files) 2024-11-17T08:16:36,874 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c34c3478acc81ba3f3c8f5caec36c13b/A in TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:36,874 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c34c3478acc81ba3f3c8f5caec36c13b/B in TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:36,875 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/5d3b66f196384e1fa6761009fe2f4d97, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/c1743033d0e9407091109eef43397391, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/eb2e64488e634039a583af91adadf73f] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp, totalSize=36.3 K 2024-11-17T08:16:36,875 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/2cccc733678e493abd4ebee27f713476, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/28bbbbda17a44465937c6a36852c27eb, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/fa66fdcec4874397b07e839f1037ecca] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp, totalSize=33.9 K 2024-11-17T08:16:36,875 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cccc733678e493abd4ebee27f713476, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731831393474 2024-11-17T08:16:36,875 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d3b66f196384e1fa6761009fe2f4d97, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731831393474 2024-11-17T08:16:36,875 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 28bbbbda17a44465937c6a36852c27eb, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731831393829 2024-11-17T08:16:36,875 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1743033d0e9407091109eef43397391, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731831393829 2024-11-17T08:16:36,875 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting fa66fdcec4874397b07e839f1037ecca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731831395099 2024-11-17T08:16:36,875 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb2e64488e634039a583af91adadf73f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731831395099 2024-11-17T08:16:36,882 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c34c3478acc81ba3f3c8f5caec36c13b#B#compaction#423 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:36,883 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/648834d0e0434d87baaefd672b5845bd is 50, key is test_row_0/B:col10/1731831395099/Put/seqid=0 2024-11-17T08:16:36,886 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c34c3478acc81ba3f3c8f5caec36c13b#A#compaction#424 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:36,886 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/7a64e1c7dc684fa58bbc40a8fe617153 is 50, key is test_row_0/A:col10/1731831395099/Put/seqid=0 2024-11-17T08:16:36,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742324_1500 (size=12949) 2024-11-17T08:16:36,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742323_1499 (size=12949) 2024-11-17T08:16:36,900 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/7a64e1c7dc684fa58bbc40a8fe617153 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/7a64e1c7dc684fa58bbc40a8fe617153 2024-11-17T08:16:36,905 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c34c3478acc81ba3f3c8f5caec36c13b/A of c34c3478acc81ba3f3c8f5caec36c13b into 7a64e1c7dc684fa58bbc40a8fe617153(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:36,905 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c34c3478acc81ba3f3c8f5caec36c13b: 2024-11-17T08:16:36,905 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., storeName=c34c3478acc81ba3f3c8f5caec36c13b/A, priority=13, startTime=1731831396874; duration=0sec 2024-11-17T08:16:36,906 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:16:36,906 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c34c3478acc81ba3f3c8f5caec36c13b:A 2024-11-17T08:16:36,906 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:16:36,906 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:16:36,907 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): c34c3478acc81ba3f3c8f5caec36c13b/C is initiating minor compaction (all files) 2024-11-17T08:16:36,907 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c34c3478acc81ba3f3c8f5caec36c13b/C in TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:36,907 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/ae7294bb9b8543f0899ee1a3c2080be4, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/9c7ec94cf2e244b3b928fd9fef0d7b17, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/a2f117896f914daf8949f3286b57d3a6] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp, totalSize=33.9 K 2024-11-17T08:16:36,907 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae7294bb9b8543f0899ee1a3c2080be4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731831393474 2024-11-17T08:16:36,907 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c7ec94cf2e244b3b928fd9fef0d7b17, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731831393829 2024-11-17T08:16:36,907 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2f117896f914daf8949f3286b57d3a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731831395099 2024-11-17T08:16:36,913 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c34c3478acc81ba3f3c8f5caec36c13b#C#compaction#425 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:36,913 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/07c47968caa247d09b6b00b7ab58be28 is 50, key is test_row_0/C:col10/1731831395099/Put/seqid=0 2024-11-17T08:16:36,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742325_1501 (size=12949) 2024-11-17T08:16:36,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-17T08:16:36,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:36,980 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing c34c3478acc81ba3f3c8f5caec36c13b 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-17T08:16:36,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=A 2024-11-17T08:16:36,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:36,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=B 2024-11-17T08:16:36,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:36,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=C 2024-11-17T08:16:36,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:36,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/90404cfa288b40069a5e9ffdf3299de1 is 50, key is test_row_0/A:col10/1731831395601/Put/seqid=0 2024-11-17T08:16:36,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742326_1502 (size=12301) 2024-11-17T08:16:37,301 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/648834d0e0434d87baaefd672b5845bd as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/648834d0e0434d87baaefd672b5845bd 2024-11-17T08:16:37,304 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c34c3478acc81ba3f3c8f5caec36c13b/B of c34c3478acc81ba3f3c8f5caec36c13b into 648834d0e0434d87baaefd672b5845bd(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:37,304 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c34c3478acc81ba3f3c8f5caec36c13b: 2024-11-17T08:16:37,304 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., storeName=c34c3478acc81ba3f3c8f5caec36c13b/B, priority=13, startTime=1731831396874; duration=0sec 2024-11-17T08:16:37,304 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:37,304 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c34c3478acc81ba3f3c8f5caec36c13b:B 2024-11-17T08:16:37,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. as already flushing 2024-11-17T08:16:37,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on c34c3478acc81ba3f3c8f5caec36c13b 2024-11-17T08:16:37,322 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/07c47968caa247d09b6b00b7ab58be28 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/07c47968caa247d09b6b00b7ab58be28 2024-11-17T08:16:37,326 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c34c3478acc81ba3f3c8f5caec36c13b/C of c34c3478acc81ba3f3c8f5caec36c13b into 07c47968caa247d09b6b00b7ab58be28(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:37,326 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c34c3478acc81ba3f3c8f5caec36c13b: 2024-11-17T08:16:37,326 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., storeName=c34c3478acc81ba3f3c8f5caec36c13b/C, priority=13, startTime=1731831396874; duration=0sec 2024-11-17T08:16:37,326 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:37,326 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c34c3478acc81ba3f3c8f5caec36c13b:C 2024-11-17T08:16:37,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T08:16:37,380 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731831397358, max timestamp=1731831397377Num uniques 859; 2024-11-17T08:16:37,380 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731831397358, max timestamp=1731831397377Num uniques 859; 2024-11-17T08:16:37,380 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T08:16:37,380 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T08:16:37,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731831397358, max timestamp=1731831397377Num uniques 859; 2024-11-17T08:16:37,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T08:16:37,390 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/90404cfa288b40069a5e9ffdf3299de1 2024-11-17T08:16:37,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/26b9b31c04564f7aaba4715a792b3e2c is 50, key is test_row_0/B:col10/1731831395601/Put/seqid=0 2024-11-17T08:16:37,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742327_1503 (size=12301) 2024-11-17T08:16:37,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:37,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831457445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:37,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:37,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:37,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:37,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:37,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831457450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:37,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:37,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:37,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:37,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:37,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831457553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:37,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:37,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:37,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:37,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:37,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831457564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:37,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:37,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:37,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:37,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:37,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831457773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:37,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:37,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831457773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:37,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:37,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:37,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:37,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:37,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831457773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:37,780 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e3d0adb87c01,40631,1731831247805 for 'test_row_1' in c34c3478acc81ba3f3c8f5caec36c13b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T08:16:37,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:37,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:37,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:37,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:37,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:37,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:37,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:37,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33940 deadline: 1731831457784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:37,793 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to e3d0adb87c01,40631,1731831247805 for 'test_row_2' in c34c3478acc81ba3f3c8f5caec36c13b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T08:16:37,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:37,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:37,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:37,798 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/26b9b31c04564f7aaba4715a792b3e2c 2024-11-17T08:16:37,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/e86bdbc5fbe148efb88ec4f3cd79ca02 is 50, key is test_row_0/C:col10/1731831395601/Put/seqid=0 2024-11-17T08:16:37,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742328_1504 (size=12301) 2024-11-17T08:16:38,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:38,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831458084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:38,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:38,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:38,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:38,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831458084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:38,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:38,090 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:38,090 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:38,090 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:38,206 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/e86bdbc5fbe148efb88ec4f3cd79ca02 2024-11-17T08:16:38,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/90404cfa288b40069a5e9ffdf3299de1 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/90404cfa288b40069a5e9ffdf3299de1 2024-11-17T08:16:38,212 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/90404cfa288b40069a5e9ffdf3299de1, entries=150, sequenceid=296, filesize=12.0 K 2024-11-17T08:16:38,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/26b9b31c04564f7aaba4715a792b3e2c as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/26b9b31c04564f7aaba4715a792b3e2c 2024-11-17T08:16:38,216 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/26b9b31c04564f7aaba4715a792b3e2c, entries=150, sequenceid=296, filesize=12.0 K 2024-11-17T08:16:38,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/e86bdbc5fbe148efb88ec4f3cd79ca02 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/e86bdbc5fbe148efb88ec4f3cd79ca02 2024-11-17T08:16:38,219 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/e86bdbc5fbe148efb88ec4f3cd79ca02, entries=150, sequenceid=296, filesize=12.0 K 2024-11-17T08:16:38,219 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for c34c3478acc81ba3f3c8f5caec36c13b in 1239ms, sequenceid=296, compaction requested=false 2024-11-17T08:16:38,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for c34c3478acc81ba3f3c8f5caec36c13b: 2024-11-17T08:16:38,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:38,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-17T08:16:38,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-17T08:16:38,222 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-17T08:16:38,222 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0040 sec 2024-11-17T08:16:38,223 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 2.0070 sec 2024-11-17T08:16:38,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T08:16:38,363 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:16:38,364 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:16:38,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-17T08:16:38,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T08:16:38,365 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:16:38,366 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:16:38,366 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:16:38,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T08:16:38,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-17T08:16:38,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:38,518 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing c34c3478acc81ba3f3c8f5caec36c13b 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-17T08:16:38,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=A 2024-11-17T08:16:38,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:16:38,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=B 2024-11-17T08:16:38,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:16:38,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=C 2024-11-17T08:16:38,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:16:38,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/6df1db918a50416aa6dec1e01be8677f is 50, key is test_row_0/A:col10/1731831397448/Put/seqid=0 2024-11-17T08:16:38,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742329_1505 (size=12301) 2024-11-17T08:16:38,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on c34c3478acc81ba3f3c8f5caec36c13b 2024-11-17T08:16:38,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. as already flushing 2024-11-17T08:16:38,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:38,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831458617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:38,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:38,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:38,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:38,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:38,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831458623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:38,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:38,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:38,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:38,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T08:16:38,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:38,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831458733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:38,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:38,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831458733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:38,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:38,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:38,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:38,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:38,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:38,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:38,926 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/6df1db918a50416aa6dec1e01be8677f 2024-11-17T08:16:38,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/a4e8ce6309bd4bd09daa4a8e6dec2ef7 is 50, key is test_row_0/B:col10/1731831397448/Put/seqid=0 2024-11-17T08:16:38,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742330_1506 (size=12301) 2024-11-17T08:16:38,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:38,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831458954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:38,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:38,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:38,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831458954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:38,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:38,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:38,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:38,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:38,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:38,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T08:16:39,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:39,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831459263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:39,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:39,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:39,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831459274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:39,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:39,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:39,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:39,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:39,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:39,335 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/a4e8ce6309bd4bd09daa4a8e6dec2ef7 2024-11-17T08:16:39,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/c36a96e06c15431aac3a3785772e99fd is 50, key is test_row_0/C:col10/1731831397448/Put/seqid=0 2024-11-17T08:16:39,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742331_1507 (size=12301) 2024-11-17T08:16:39,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T08:16:39,745 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/c36a96e06c15431aac3a3785772e99fd 2024-11-17T08:16:39,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/6df1db918a50416aa6dec1e01be8677f as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/6df1db918a50416aa6dec1e01be8677f 2024-11-17T08:16:39,756 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/6df1db918a50416aa6dec1e01be8677f, entries=150, sequenceid=335, filesize=12.0 K 2024-11-17T08:16:39,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/a4e8ce6309bd4bd09daa4a8e6dec2ef7 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/a4e8ce6309bd4bd09daa4a8e6dec2ef7 2024-11-17T08:16:39,760 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/a4e8ce6309bd4bd09daa4a8e6dec2ef7, entries=150, sequenceid=335, filesize=12.0 K 2024-11-17T08:16:39,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/c36a96e06c15431aac3a3785772e99fd as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/c36a96e06c15431aac3a3785772e99fd 2024-11-17T08:16:39,764 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/c36a96e06c15431aac3a3785772e99fd, entries=150, sequenceid=335, filesize=12.0 K 2024-11-17T08:16:39,764 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for c34c3478acc81ba3f3c8f5caec36c13b in 1246ms, sequenceid=335, compaction requested=true 2024-11-17T08:16:39,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for c34c3478acc81ba3f3c8f5caec36c13b: 2024-11-17T08:16:39,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:39,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-17T08:16:39,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-17T08:16:39,768 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-17T08:16:39,768 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4000 sec 2024-11-17T08:16:39,769 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.4050 sec 2024-11-17T08:16:39,791 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c34c3478acc81ba3f3c8f5caec36c13b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T08:16:39,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=A 2024-11-17T08:16:39,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:39,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=B 2024-11-17T08:16:39,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:39,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=C 2024-11-17T08:16:39,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:39,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on c34c3478acc81ba3f3c8f5caec36c13b 2024-11-17T08:16:39,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/7799b0a73bb548eca38620fc3352ed62 is 50, key is test_row_0/A:col10/1731831399790/Put/seqid=0 2024-11-17T08:16:39,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742332_1508 (size=14741) 2024-11-17T08:16:39,870 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731831399849, max timestamp=1731831399866Num uniques 859; 2024-11-17T08:16:39,870 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T08:16:39,870 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731831399849, max timestamp=1731831399866Num uniques 859; 2024-11-17T08:16:39,870 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T08:16:39,873 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731831399849, max timestamp=1731831399866Num uniques 859; 2024-11-17T08:16:39,873 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T08:16:39,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:39,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831459914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:39,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:39,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:39,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:39,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831459919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:39,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:39,929 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:39,929 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:39,929 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:39,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831459920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:39,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:39,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:39,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33940 deadline: 1731831459921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:39,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:39,936 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:39,936 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:39,936 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:40,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831460034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:40,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:40,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:40,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831460034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:40,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:40,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:40,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:40,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:40,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831460043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:40,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:40,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:40,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:40,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33940 deadline: 1731831460043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:40,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:40,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:40,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:40,204 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/7799b0a73bb548eca38620fc3352ed62 2024-11-17T08:16:40,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/70d3e844d7d1432b8afa91b0fa428e3b is 50, key is test_row_0/B:col10/1731831399790/Put/seqid=0 2024-11-17T08:16:40,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742333_1509 (size=12301) 2024-11-17T08:16:40,232 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/70d3e844d7d1432b8afa91b0fa428e3b 2024-11-17T08:16:40,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/39e0adf048914e5aa99816ab90a4daea is 50, key is test_row_0/C:col10/1731831399790/Put/seqid=0 2024-11-17T08:16:40,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742334_1510 (size=12301) 2024-11-17T08:16:40,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/39e0adf048914e5aa99816ab90a4daea 2024-11-17T08:16:40,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:40,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33930 deadline: 1731831460253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:40,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:40,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:40,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:40,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:40,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831460253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:40,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:40,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:40,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:40,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/7799b0a73bb548eca38620fc3352ed62 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/7799b0a73bb548eca38620fc3352ed62 2024-11-17T08:16:40,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/7799b0a73bb548eca38620fc3352ed62, entries=200, sequenceid=345, filesize=14.4 K 2024-11-17T08:16:40,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/70d3e844d7d1432b8afa91b0fa428e3b as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/70d3e844d7d1432b8afa91b0fa428e3b 2024-11-17T08:16:40,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/70d3e844d7d1432b8afa91b0fa428e3b, entries=150, sequenceid=345, filesize=12.0 K 2024-11-17T08:16:40,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/39e0adf048914e5aa99816ab90a4daea as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/39e0adf048914e5aa99816ab90a4daea 2024-11-17T08:16:40,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:40,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33900 deadline: 1731831460264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:40,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:40,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:40,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:40,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:40,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33940 deadline: 1731831460265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:40,282 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:40,282 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:40,282 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:40,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/39e0adf048914e5aa99816ab90a4daea, entries=150, sequenceid=345, filesize=12.0 K 2024-11-17T08:16:40,285 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for c34c3478acc81ba3f3c8f5caec36c13b in 494ms, sequenceid=345, compaction requested=true 2024-11-17T08:16:40,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c34c3478acc81ba3f3c8f5caec36c13b: 2024-11-17T08:16:40,285 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T08:16:40,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c34c3478acc81ba3f3c8f5caec36c13b:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:16:40,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:40,285 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T08:16:40,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c34c3478acc81ba3f3c8f5caec36c13b:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:16:40,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:40,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c34c3478acc81ba3f3c8f5caec36c13b:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:16:40,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:16:40,287 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T08:16:40,287 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T08:16:40,287 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): c34c3478acc81ba3f3c8f5caec36c13b/B is initiating minor compaction (all files) 2024-11-17T08:16:40,287 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): c34c3478acc81ba3f3c8f5caec36c13b/A is initiating minor compaction (all files) 2024-11-17T08:16:40,287 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c34c3478acc81ba3f3c8f5caec36c13b/A in TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:40,287 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c34c3478acc81ba3f3c8f5caec36c13b/B in TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:40,287 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/648834d0e0434d87baaefd672b5845bd, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/26b9b31c04564f7aaba4715a792b3e2c, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/a4e8ce6309bd4bd09daa4a8e6dec2ef7, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/70d3e844d7d1432b8afa91b0fa428e3b] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp, totalSize=48.7 K 2024-11-17T08:16:40,287 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/7a64e1c7dc684fa58bbc40a8fe617153, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/90404cfa288b40069a5e9ffdf3299de1, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/6df1db918a50416aa6dec1e01be8677f, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/7799b0a73bb548eca38620fc3352ed62] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp, totalSize=51.1 K 2024-11-17T08:16:40,287 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a64e1c7dc684fa58bbc40a8fe617153, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731831395099 2024-11-17T08:16:40,287 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 648834d0e0434d87baaefd672b5845bd, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731831395099 2024-11-17T08:16:40,288 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 26b9b31c04564f7aaba4715a792b3e2c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731831395601 2024-11-17T08:16:40,288 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90404cfa288b40069a5e9ffdf3299de1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731831395601 2024-11-17T08:16:40,288 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting a4e8ce6309bd4bd09daa4a8e6dec2ef7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731831397430 2024-11-17T08:16:40,288 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6df1db918a50416aa6dec1e01be8677f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731831397430 2024-11-17T08:16:40,288 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 70d3e844d7d1432b8afa91b0fa428e3b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731831398610 2024-11-17T08:16:40,288 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7799b0a73bb548eca38620fc3352ed62, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731831398610 2024-11-17T08:16:40,297 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c34c3478acc81ba3f3c8f5caec36c13b#A#compaction#435 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:40,298 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/c0efbeeeb10b4ac7b20265c21ba00136 is 50, key is test_row_0/A:col10/1731831399790/Put/seqid=0 2024-11-17T08:16:40,301 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c34c3478acc81ba3f3c8f5caec36c13b#B#compaction#436 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:40,301 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/d7ee844507a74b43928da77edf429b81 is 50, key is test_row_0/B:col10/1731831399790/Put/seqid=0 2024-11-17T08:16:40,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742335_1511 (size=13085) 2024-11-17T08:16:40,322 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/d7ee844507a74b43928da77edf429b81 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/d7ee844507a74b43928da77edf429b81 2024-11-17T08:16:40,330 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c34c3478acc81ba3f3c8f5caec36c13b/B of c34c3478acc81ba3f3c8f5caec36c13b into d7ee844507a74b43928da77edf429b81(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:40,330 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c34c3478acc81ba3f3c8f5caec36c13b: 2024-11-17T08:16:40,330 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., storeName=c34c3478acc81ba3f3c8f5caec36c13b/B, priority=12, startTime=1731831400285; duration=0sec 2024-11-17T08:16:40,330 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:16:40,330 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c34c3478acc81ba3f3c8f5caec36c13b:B 2024-11-17T08:16:40,330 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T08:16:40,331 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T08:16:40,332 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): c34c3478acc81ba3f3c8f5caec36c13b/C is initiating minor compaction (all files) 2024-11-17T08:16:40,332 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c34c3478acc81ba3f3c8f5caec36c13b/C in TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:40,332 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/07c47968caa247d09b6b00b7ab58be28, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/e86bdbc5fbe148efb88ec4f3cd79ca02, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/c36a96e06c15431aac3a3785772e99fd, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/39e0adf048914e5aa99816ab90a4daea] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp, totalSize=48.7 K 2024-11-17T08:16:40,332 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 07c47968caa247d09b6b00b7ab58be28, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731831395099 2024-11-17T08:16:40,335 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting e86bdbc5fbe148efb88ec4f3cd79ca02, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731831395601 2024-11-17T08:16:40,336 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting c36a96e06c15431aac3a3785772e99fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731831397430 2024-11-17T08:16:40,336 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 39e0adf048914e5aa99816ab90a4daea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731831398610 2024-11-17T08:16:40,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742336_1512 (size=13085) 2024-11-17T08:16:40,341 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/c0efbeeeb10b4ac7b20265c21ba00136 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/c0efbeeeb10b4ac7b20265c21ba00136 2024-11-17T08:16:40,347 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c34c3478acc81ba3f3c8f5caec36c13b/A of c34c3478acc81ba3f3c8f5caec36c13b into c0efbeeeb10b4ac7b20265c21ba00136(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:40,347 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c34c3478acc81ba3f3c8f5caec36c13b: 2024-11-17T08:16:40,347 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., storeName=c34c3478acc81ba3f3c8f5caec36c13b/A, priority=12, startTime=1731831400285; duration=0sec 2024-11-17T08:16:40,347 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:40,347 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c34c3478acc81ba3f3c8f5caec36c13b:A 2024-11-17T08:16:40,350 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c34c3478acc81ba3f3c8f5caec36c13b#C#compaction#437 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:40,350 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/65963cf9dfd44198ba0057573d58e09a is 50, key is test_row_0/C:col10/1731831399790/Put/seqid=0 2024-11-17T08:16:40,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742337_1513 (size=13085) 2024-11-17T08:16:40,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T08:16:40,515 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:16:40,515 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:16:40,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-17T08:16:40,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-17T08:16:40,517 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:16:40,518 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:16:40,518 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:16:40,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on c34c3478acc81ba3f3c8f5caec36c13b 2024-11-17T08:16:40,582 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731831399913, max timestamp=1731831400574Num uniques 1018; 2024-11-17T08:16:40,582 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731831399913, max timestamp=1731831400574Num uniques 1018; 2024-11-17T08:16:40,582 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-17T08:16:40,582 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-17T08:16:40,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c34c3478acc81ba3f3c8f5caec36c13b 3/3 column families, dataSize=64.85 KB heapSize=178.80 KB 2024-11-17T08:16:40,583 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731831399913, max timestamp=1731831400574Num uniques 1018; 2024-11-17T08:16:40,583 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-17T08:16:40,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=A 2024-11-17T08:16:40,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:16:40,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=B 2024-11-17T08:16:40,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:16:40,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=C 2024-11-17T08:16:40,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:16:40,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/b628f058596a48b19ae59fa6e0ac4a2b is 50, key is test_row_0/A:col10/1731831400580/Put/seqid=0 2024-11-17T08:16:40,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-17T08:16:40,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742338_1514 (size=17181) 2024-11-17T08:16:40,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731831400637, max timestamp=1731831400651Num uniques 859; 2024-11-17T08:16:40,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731831400637, max timestamp=1731831400651Num uniques 859; 2024-11-17T08:16:40,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731831400637, max timestamp=1731831400651Num uniques 859; 2024-11-17T08:16:40,655 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T08:16:40,655 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T08:16:40,655 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T08:16:40,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-17T08:16:40,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:40,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. as already flushing 2024-11-17T08:16:40,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:40,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:40,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:40,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:40,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731831400700, max timestamp=1731831400720Num uniques 1018; 2024-11-17T08:16:40,727 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-17T08:16:40,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731831400700, max timestamp=1731831400720Num uniques 1018; 2024-11-17T08:16:40,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-17T08:16:40,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731831400700, max timestamp=1731831400720Num uniques 1018; 2024-11-17T08:16:40,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-17T08:16:40,770 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/65963cf9dfd44198ba0057573d58e09a as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/65963cf9dfd44198ba0057573d58e09a 2024-11-17T08:16:40,775 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c34c3478acc81ba3f3c8f5caec36c13b/C of c34c3478acc81ba3f3c8f5caec36c13b into 65963cf9dfd44198ba0057573d58e09a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:40,775 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c34c3478acc81ba3f3c8f5caec36c13b: 2024-11-17T08:16:40,775 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., storeName=c34c3478acc81ba3f3c8f5caec36c13b/C, priority=12, startTime=1731831400285; duration=0sec 2024-11-17T08:16:40,775 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:40,775 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c34c3478acc81ba3f3c8f5caec36c13b:C 2024-11-17T08:16:40,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1731831400768, max timestamp=1731831400792Num uniques 1077; 2024-11-17T08:16:40,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=14972, suffix heap size=139976,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=327 2024-11-17T08:16:40,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1731831400768, max timestamp=1731831400792Num uniques 1077; 2024-11-17T08:16:40,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=14972, suffix heap size=139976,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=327 2024-11-17T08:16:40,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731831400768, max timestamp=1731831400792Num uniques 1077; 2024-11-17T08:16:40,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-17T08:16:40,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-17T08:16:40,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:40,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. as already flushing 2024-11-17T08:16:40,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:40,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:40,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:40,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:40,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-17T08:16:40,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731831400825, max timestamp=1731831400859Num uniques 1186; 2024-11-17T08:16:40,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=10806, suffix heap size=151312,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=236 2024-11-17T08:16:40,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731831400825, max timestamp=1731831400859Num uniques 1186; 2024-11-17T08:16:40,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=10806, suffix heap size=151312,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=236 2024-11-17T08:16:40,867 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731831400825, max timestamp=1731831400859Num uniques 1086; 2024-11-17T08:16:40,867 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-17T08:16:40,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11210, totalHeapSize=25720, min timestamp=1731831400901, max timestamp=1731831400936Num uniques 1095; 2024-11-17T08:16:40,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50140, new segment data size=11210, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-17T08:16:40,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731831400912, max timestamp=1731831400936Num uniques 1095; 2024-11-17T08:16:40,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=6870, suffix heap size=141848,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=150 2024-11-17T08:16:40,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731831400912, max timestamp=1731831400936Num uniques 1095; 2024-11-17T08:16:40,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50149, new segment data size=6870, suffix heap size=141848,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=150 2024-11-17T08:16:40,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-17T08:16:40,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:40,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. as already flushing 2024-11-17T08:16:40,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:40,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:40,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:40,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:41,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731831401004, max timestamp=1731831401016Num uniques 1009; 2024-11-17T08:16:41,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46211, new segment data size=9340, suffix heap size=132904,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=204 2024-11-17T08:16:41,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731831401004, max timestamp=1731831401016Num uniques 1009; 2024-11-17T08:16:41,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46210, new segment data size=9340, suffix heap size=132904,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=204 2024-11-17T08:16:41,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731831401004, max timestamp=1731831401016Num uniques 1104; 2024-11-17T08:16:41,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-17T08:16:41,032 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/b628f058596a48b19ae59fa6e0ac4a2b 2024-11-17T08:16:41,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/2353f0ee151843189d84468ef41b4453 is 50, key is test_row_0/B:col10/1731831400580/Put/seqid=0 2024-11-17T08:16:41,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742339_1515 (size=12301) 2024-11-17T08:16:41,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-17T08:16:41,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:41,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. as already flushing 2024-11-17T08:16:41,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:41,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:41,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:41,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:41,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:41,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33876 deadline: 1731831461116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:41,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 , the old value is region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:41,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:41,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., hostname=e3d0adb87c01,40631,1731831247805, seqNum=2 because the exception is null or not the one we care about 2024-11-17T08:16:41,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731831401107, max timestamp=1731831401131Num uniques 1063; 2024-11-17T08:16:41,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731831401107, max timestamp=1731831401131Num uniques 1063; 2024-11-17T08:16:41,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-17T08:16:41,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-17T08:16:41,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731831401107, max timestamp=1731831401131Num uniques 1063; 2024-11-17T08:16:41,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-17T08:16:41,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-17T08:16:41,274 INFO [Thread-1997 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1997. 2024-11-17T08:16:41,274 DEBUG [Thread-1997 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:16:41,274 DEBUG [Thread-1997 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:41,274 DEBUG [Thread-1997 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:41,274 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:16:41,275 INFO [Thread-1986 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1986. 2024-11-17T08:16:41,276 DEBUG [Thread-1986 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:16:41,276 DEBUG [Thread-1986 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:41,276 DEBUG [Thread-1986 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:41,276 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:16:41,276 INFO [Thread-1994 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1994. 2024-11-17T08:16:41,276 DEBUG [Thread-1994 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:16:41,276 DEBUG [Thread-1994 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:41,276 DEBUG [Thread-1994 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:41,276 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:16:41,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731831401228, max timestamp=1731831401276Num uniques 1022; 2024-11-17T08:16:41,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=12452, suffix heap size=134256,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=272 2024-11-17T08:16:41,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731831401228, max timestamp=1731831401276Num uniques 1022; 2024-11-17T08:16:41,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=12452, suffix heap size=134256,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=272 2024-11-17T08:16:41,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12447, totalHeapSize=28528, min timestamp=1731831401228, max timestamp=1731831401276Num uniques 1022; 2024-11-17T08:16:41,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46797, new segment data size=12447, suffix heap size=134256,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=272 2024-11-17T08:16:41,282 INFO [Thread-1990 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1990. 2024-11-17T08:16:41,282 DEBUG [Thread-1990 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:16:41,282 DEBUG [Thread-1990 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:41,282 DEBUG [Thread-1990 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:41,282 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:16:41,283 INFO [Thread-1992 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1992. 2024-11-17T08:16:41,283 DEBUG [Thread-1992 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:16:41,283 DEBUG [Thread-1992 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:41,283 DEBUG [Thread-1992 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:41,283 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:16:41,288 INFO [Thread-1999 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1999. 2024-11-17T08:16:41,288 DEBUG [Thread-1999 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:16:41,288 DEBUG [Thread-1999 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:41,288 DEBUG [Thread-1999 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:41,288 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:16:41,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-17T08:16:41,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:41,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. as already flushing 2024-11-17T08:16:41,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:41,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:41,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:41,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:41,296 INFO [Thread-2001 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2001. 2024-11-17T08:16:41,296 DEBUG [Thread-2001 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:16:41,296 DEBUG [Thread-2001 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:41,296 DEBUG [Thread-2001 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:41,296 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:16:41,297 INFO [Thread-2003 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2003. 2024-11-17T08:16:41,297 DEBUG [Thread-2003 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:16:41,297 DEBUG [Thread-2003 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:41,297 INFO [Thread-2005 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2005. 2024-11-17T08:16:41,297 DEBUG [Thread-2003 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:41,297 DEBUG [Thread-2005 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:16:41,297 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:16:41,297 DEBUG [Thread-2005 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:41,297 DEBUG [Thread-2005 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:41,297 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:16:41,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-17T08:16:41,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:41,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. as already flushing 2024-11-17T08:16:41,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:41,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:41,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:41,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:41,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/2353f0ee151843189d84468ef41b4453 2024-11-17T08:16:41,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/ebcc6fb9d02445dc963bad05ae48eb58 is 50, key is test_row_0/C:col10/1731831400580/Put/seqid=0 2024-11-17T08:16:41,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742340_1516 (size=12301) 2024-11-17T08:16:41,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/ebcc6fb9d02445dc963bad05ae48eb58 2024-11-17T08:16:41,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/b628f058596a48b19ae59fa6e0ac4a2b as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/b628f058596a48b19ae59fa6e0ac4a2b 2024-11-17T08:16:41,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/b628f058596a48b19ae59fa6e0ac4a2b, entries=250, sequenceid=386, filesize=16.8 K 2024-11-17T08:16:41,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/2353f0ee151843189d84468ef41b4453 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/2353f0ee151843189d84468ef41b4453 2024-11-17T08:16:41,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/2353f0ee151843189d84468ef41b4453, entries=150, sequenceid=386, filesize=12.0 K 2024-11-17T08:16:41,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/ebcc6fb9d02445dc963bad05ae48eb58 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/ebcc6fb9d02445dc963bad05ae48eb58 2024-11-17T08:16:41,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/ebcc6fb9d02445dc963bad05ae48eb58, entries=150, sequenceid=386, filesize=12.0 K 2024-11-17T08:16:41,485 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~85.27 KB/87312, currentSize=46.96 KB/48090 for c34c3478acc81ba3f3c8f5caec36c13b in 903ms, sequenceid=386, compaction requested=false 2024-11-17T08:16:41,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c34c3478acc81ba3f3c8f5caec36c13b: 2024-11-17T08:16:41,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-17T08:16:41,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:41,602 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing c34c3478acc81ba3f3c8f5caec36c13b 3/3 column families, dataSize=46.96 KB heapSize=115.41 KB 2024-11-17T08:16:41,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=A 2024-11-17T08:16:41,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:16:41,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=B 2024-11-17T08:16:41,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:16:41,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=C 2024-11-17T08:16:41,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:16:41,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/4590002d152e4435a2ab3cd8dfae52cc is 50, key is test_row_0/A:col10/1731831401275/Put/seqid=0 2024-11-17T08:16:41,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742341_1517 (size=12301) 2024-11-17T08:16:41,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-17T08:16:42,010 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/4590002d152e4435a2ab3cd8dfae52cc 2024-11-17T08:16:42,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/ef9fbcd634864388adba1c628d2a2982 is 50, key is test_row_0/B:col10/1731831401275/Put/seqid=0 2024-11-17T08:16:42,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742342_1518 (size=12301) 2024-11-17T08:16:42,421 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/ef9fbcd634864388adba1c628d2a2982 2024-11-17T08:16:42,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/54a7609160134a5fa2180ca280ac950a is 50, key is test_row_0/C:col10/1731831401275/Put/seqid=0 2024-11-17T08:16:42,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742343_1519 (size=12301) 2024-11-17T08:16:42,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-17T08:16:42,841 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/54a7609160134a5fa2180ca280ac950a 2024-11-17T08:16:42,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/4590002d152e4435a2ab3cd8dfae52cc as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/4590002d152e4435a2ab3cd8dfae52cc 2024-11-17T08:16:42,848 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/4590002d152e4435a2ab3cd8dfae52cc, entries=150, sequenceid=529, filesize=12.0 K 2024-11-17T08:16:42,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/ef9fbcd634864388adba1c628d2a2982 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/ef9fbcd634864388adba1c628d2a2982 2024-11-17T08:16:42,851 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/ef9fbcd634864388adba1c628d2a2982, entries=150, sequenceid=529, filesize=12.0 K 2024-11-17T08:16:42,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/54a7609160134a5fa2180ca280ac950a as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/54a7609160134a5fa2180ca280ac950a 2024-11-17T08:16:42,854 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/54a7609160134a5fa2180ca280ac950a, entries=150, sequenceid=529, filesize=12.0 K 2024-11-17T08:16:42,855 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~115.36 KB/118128, currentSize=0 B/0 for c34c3478acc81ba3f3c8f5caec36c13b in 1253ms, sequenceid=529, compaction requested=true 2024-11-17T08:16:42,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for c34c3478acc81ba3f3c8f5caec36c13b: 2024-11-17T08:16:42,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:42,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-17T08:16:42,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-17T08:16:42,858 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-17T08:16:42,858 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3380 sec 2024-11-17T08:16:42,859 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 2.3430 sec 2024-11-17T08:16:44,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-17T08:16:44,695 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:16:46,155 INFO [Thread-1988 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1988. 2024-11-17T08:16:46,155 DEBUG [Thread-1988 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:16:46,156 DEBUG [Thread-1988 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:46,156 DEBUG [Thread-1988 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:46,156 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:16:46,156 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T08:16:46,156 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 119 2024-11-17T08:16:46,156 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-11-17T08:16:46,156 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-17T08:16:46,156 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-17T08:16:46,156 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 139 2024-11-17T08:16:46,156 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T08:16:46,156 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T08:16:46,156 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1201 2024-11-17T08:16:46,156 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3602 rows 2024-11-17T08:16:46,156 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1212 2024-11-17T08:16:46,156 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3630 rows 2024-11-17T08:16:46,156 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1206 2024-11-17T08:16:46,156 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3617 rows 2024-11-17T08:16:46,156 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1199 2024-11-17T08:16:46,156 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3596 rows 2024-11-17T08:16:46,156 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1201 2024-11-17T08:16:46,156 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3602 rows 2024-11-17T08:16:46,156 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T08:16:46,156 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T08:16:46,156 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:46,156 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:46,156 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:16:46,157 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-17T08:16:46,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T08:16:46,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-17T08:16:46,159 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731831406159"}]},"ts":"1731831406159"} 2024-11-17T08:16:46,160 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T08:16:46,160 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T08:16:46,161 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T08:16:46,162 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c34c3478acc81ba3f3c8f5caec36c13b, UNASSIGN}] 2024-11-17T08:16:46,163 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c34c3478acc81ba3f3c8f5caec36c13b, UNASSIGN 2024-11-17T08:16:46,163 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=c34c3478acc81ba3f3c8f5caec36c13b, regionState=CLOSING, regionLocation=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:46,164 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c34c3478acc81ba3f3c8f5caec36c13b, UNASSIGN because future has completed 2024-11-17T08:16:46,164 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T08:16:46,165 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805}] 2024-11-17T08:16:46,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-17T08:16:46,317 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close c34c3478acc81ba3f3c8f5caec36c13b 2024-11-17T08:16:46,317 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T08:16:46,317 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing c34c3478acc81ba3f3c8f5caec36c13b, disabling compactions & flushes 2024-11-17T08:16:46,317 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:46,317 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:46,317 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. after waiting 0 ms 2024-11-17T08:16:46,317 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:46,317 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(2902): Flushing c34c3478acc81ba3f3c8f5caec36c13b 3/3 column families, dataSize=6.71 KB heapSize=20.67 KB 2024-11-17T08:16:46,317 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=A 2024-11-17T08:16:46,317 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:46,317 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=B 2024-11-17T08:16:46,317 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:46,317 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c34c3478acc81ba3f3c8f5caec36c13b, store=C 2024-11-17T08:16:46,317 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:46,320 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/28fb9839bc2c4fc78541ac3bd4cb6bc8 is 50, key is test_row_2/A:col10/1731831406154/Put/seqid=0 2024-11-17T08:16:46,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742344_1520 (size=7415) 2024-11-17T08:16:46,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-17T08:16:46,724 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=533 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/28fb9839bc2c4fc78541ac3bd4cb6bc8 2024-11-17T08:16:46,730 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/d0ccdbccd42943d28f321c37512fdc80 is 50, key is test_row_2/B:col10/1731831406154/Put/seqid=0 2024-11-17T08:16:46,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742345_1521 (size=7415) 2024-11-17T08:16:46,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-17T08:16:47,134 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=533 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/d0ccdbccd42943d28f321c37512fdc80 2024-11-17T08:16:47,141 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/9fac5f42dcc1469684233c1b8cfb8893 is 50, key is test_row_2/C:col10/1731831406154/Put/seqid=0 2024-11-17T08:16:47,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742346_1522 (size=7415) 2024-11-17T08:16:47,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-17T08:16:47,546 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=533 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/9fac5f42dcc1469684233c1b8cfb8893 2024-11-17T08:16:47,551 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/A/28fb9839bc2c4fc78541ac3bd4cb6bc8 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/28fb9839bc2c4fc78541ac3bd4cb6bc8 2024-11-17T08:16:47,555 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/28fb9839bc2c4fc78541ac3bd4cb6bc8, entries=50, sequenceid=533, filesize=7.2 K 2024-11-17T08:16:47,556 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/B/d0ccdbccd42943d28f321c37512fdc80 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/d0ccdbccd42943d28f321c37512fdc80 2024-11-17T08:16:47,560 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/d0ccdbccd42943d28f321c37512fdc80, entries=50, sequenceid=533, filesize=7.2 K 2024-11-17T08:16:47,561 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/.tmp/C/9fac5f42dcc1469684233c1b8cfb8893 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/9fac5f42dcc1469684233c1b8cfb8893 2024-11-17T08:16:47,565 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/9fac5f42dcc1469684233c1b8cfb8893, entries=50, sequenceid=533, filesize=7.2 K 2024-11-17T08:16:47,565 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for c34c3478acc81ba3f3c8f5caec36c13b in 1248ms, sequenceid=533, compaction requested=true 2024-11-17T08:16:47,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/c87b5052e8234c21bdbf3da7f3b8e178, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/0199baa9df41480fb40b5902761ec907, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/9b3916429b2d4169a12547d9dac4c7e1, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/467b501d7a4a4c5fb118e9cf93147720, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/bf7e3059796e4cf7b082026fb25a7585, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/73c000c521d74973b1a43a3311313aed, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/39ef1982fd5841d4ab24daba698d255e, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/e9055076ea234762a747f05593e45962, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/f6e98abedc8c4738abfd3b750295cf01, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/2799885dff3448af82c85571983bce09, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/f287d38d590b43fd83feada0ccff1ea4, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/c89d961a628f424caf1b3e8b01bd2185, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/871b560ff2d642a7a2f227fafbe8ccdf, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/03a31717b0ae456692787c0b3ebde4c6, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/d614ca6ba8c94756b71e769130774810, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/7ee73e82d4d445a6b7835d6a6a832d5a, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/5d3b66f196384e1fa6761009fe2f4d97, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/c1743033d0e9407091109eef43397391, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/eb2e64488e634039a583af91adadf73f, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/7a64e1c7dc684fa58bbc40a8fe617153, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/90404cfa288b40069a5e9ffdf3299de1, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/6df1db918a50416aa6dec1e01be8677f, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/7799b0a73bb548eca38620fc3352ed62] to archive 2024-11-17T08:16:47,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T08:16:47,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/c87b5052e8234c21bdbf3da7f3b8e178 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/c87b5052e8234c21bdbf3da7f3b8e178 2024-11-17T08:16:47,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/0199baa9df41480fb40b5902761ec907 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/0199baa9df41480fb40b5902761ec907 2024-11-17T08:16:47,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/9b3916429b2d4169a12547d9dac4c7e1 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/9b3916429b2d4169a12547d9dac4c7e1 2024-11-17T08:16:47,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/467b501d7a4a4c5fb118e9cf93147720 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/467b501d7a4a4c5fb118e9cf93147720 2024-11-17T08:16:47,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/bf7e3059796e4cf7b082026fb25a7585 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/bf7e3059796e4cf7b082026fb25a7585 2024-11-17T08:16:47,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/73c000c521d74973b1a43a3311313aed to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/73c000c521d74973b1a43a3311313aed 2024-11-17T08:16:47,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/39ef1982fd5841d4ab24daba698d255e to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/39ef1982fd5841d4ab24daba698d255e 2024-11-17T08:16:47,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/e9055076ea234762a747f05593e45962 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/e9055076ea234762a747f05593e45962 2024-11-17T08:16:47,580 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/f6e98abedc8c4738abfd3b750295cf01 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/f6e98abedc8c4738abfd3b750295cf01 2024-11-17T08:16:47,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/2799885dff3448af82c85571983bce09 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/2799885dff3448af82c85571983bce09 2024-11-17T08:16:47,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/f287d38d590b43fd83feada0ccff1ea4 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/f287d38d590b43fd83feada0ccff1ea4 2024-11-17T08:16:47,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/c89d961a628f424caf1b3e8b01bd2185 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/c89d961a628f424caf1b3e8b01bd2185 2024-11-17T08:16:47,585 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/871b560ff2d642a7a2f227fafbe8ccdf to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/871b560ff2d642a7a2f227fafbe8ccdf 2024-11-17T08:16:47,586 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/03a31717b0ae456692787c0b3ebde4c6 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/03a31717b0ae456692787c0b3ebde4c6 2024-11-17T08:16:47,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/d614ca6ba8c94756b71e769130774810 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/d614ca6ba8c94756b71e769130774810 2024-11-17T08:16:47,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/7ee73e82d4d445a6b7835d6a6a832d5a to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/7ee73e82d4d445a6b7835d6a6a832d5a 2024-11-17T08:16:47,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/5d3b66f196384e1fa6761009fe2f4d97 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/5d3b66f196384e1fa6761009fe2f4d97 2024-11-17T08:16:47,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/c1743033d0e9407091109eef43397391 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/c1743033d0e9407091109eef43397391 2024-11-17T08:16:47,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/eb2e64488e634039a583af91adadf73f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/eb2e64488e634039a583af91adadf73f 2024-11-17T08:16:47,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/7a64e1c7dc684fa58bbc40a8fe617153 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/7a64e1c7dc684fa58bbc40a8fe617153 2024-11-17T08:16:47,594 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/90404cfa288b40069a5e9ffdf3299de1 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/90404cfa288b40069a5e9ffdf3299de1 2024-11-17T08:16:47,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/6df1db918a50416aa6dec1e01be8677f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/6df1db918a50416aa6dec1e01be8677f 2024-11-17T08:16:47,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/7799b0a73bb548eca38620fc3352ed62 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/7799b0a73bb548eca38620fc3352ed62 2024-11-17T08:16:47,597 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/7222f61b3885491299633857026bf526, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/ed3f75cff5274e848a8424d4a4bfb511, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/895e94abfb53409cab7d81e41c33a377, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/358af8cd766e4ea5bead05d441f94556, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/70a753a14d214953bcaf2b6f92cec342, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/d8612083b1774a49a9ad83a081d79e14, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/00145a702f9a4af9b87a1445bae4d11a, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/47ce2268ce0247eeadaeff11d9f0a9e7, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/30b1e7adf47b457c8d172d7c841b38e0, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/a429a8019ea746f2bd71f7b9469460c2, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/bca29fa91606434d9d4eb6ffd1c277e0, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/31f7710a86bd4949a44328e9e67485e0, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/489d211e73ef4117b402648a7d4dfe29, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/e38b70bf1c3b4e81a9a008a18292c32f, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/4e8e7f28b4394274b68ff3b63072ff2c, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/2cccc733678e493abd4ebee27f713476, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/73d3437d1cd1416da35ba54ba7419f5d, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/28bbbbda17a44465937c6a36852c27eb, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/648834d0e0434d87baaefd672b5845bd, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/fa66fdcec4874397b07e839f1037ecca, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/26b9b31c04564f7aaba4715a792b3e2c, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/a4e8ce6309bd4bd09daa4a8e6dec2ef7, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/70d3e844d7d1432b8afa91b0fa428e3b] to archive 2024-11-17T08:16:47,597 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T08:16:47,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/7222f61b3885491299633857026bf526 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/7222f61b3885491299633857026bf526 2024-11-17T08:16:47,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/ed3f75cff5274e848a8424d4a4bfb511 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/ed3f75cff5274e848a8424d4a4bfb511 2024-11-17T08:16:47,600 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/895e94abfb53409cab7d81e41c33a377 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/895e94abfb53409cab7d81e41c33a377 2024-11-17T08:16:47,601 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/358af8cd766e4ea5bead05d441f94556 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/358af8cd766e4ea5bead05d441f94556 2024-11-17T08:16:47,601 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/70a753a14d214953bcaf2b6f92cec342 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/70a753a14d214953bcaf2b6f92cec342 2024-11-17T08:16:47,602 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/d8612083b1774a49a9ad83a081d79e14 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/d8612083b1774a49a9ad83a081d79e14 2024-11-17T08:16:47,603 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/00145a702f9a4af9b87a1445bae4d11a to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/00145a702f9a4af9b87a1445bae4d11a 2024-11-17T08:16:47,604 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/47ce2268ce0247eeadaeff11d9f0a9e7 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/47ce2268ce0247eeadaeff11d9f0a9e7 2024-11-17T08:16:47,604 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/30b1e7adf47b457c8d172d7c841b38e0 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/30b1e7adf47b457c8d172d7c841b38e0 2024-11-17T08:16:47,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/a429a8019ea746f2bd71f7b9469460c2 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/a429a8019ea746f2bd71f7b9469460c2 2024-11-17T08:16:47,606 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/bca29fa91606434d9d4eb6ffd1c277e0 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/bca29fa91606434d9d4eb6ffd1c277e0 2024-11-17T08:16:47,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/31f7710a86bd4949a44328e9e67485e0 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/31f7710a86bd4949a44328e9e67485e0 2024-11-17T08:16:47,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/489d211e73ef4117b402648a7d4dfe29 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/489d211e73ef4117b402648a7d4dfe29 2024-11-17T08:16:47,608 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/e38b70bf1c3b4e81a9a008a18292c32f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/e38b70bf1c3b4e81a9a008a18292c32f 2024-11-17T08:16:47,609 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/4e8e7f28b4394274b68ff3b63072ff2c to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/4e8e7f28b4394274b68ff3b63072ff2c 2024-11-17T08:16:47,610 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/2cccc733678e493abd4ebee27f713476 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/2cccc733678e493abd4ebee27f713476 2024-11-17T08:16:47,610 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/73d3437d1cd1416da35ba54ba7419f5d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/73d3437d1cd1416da35ba54ba7419f5d 2024-11-17T08:16:47,611 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/28bbbbda17a44465937c6a36852c27eb to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/28bbbbda17a44465937c6a36852c27eb 2024-11-17T08:16:47,612 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/648834d0e0434d87baaefd672b5845bd to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/648834d0e0434d87baaefd672b5845bd 2024-11-17T08:16:47,613 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/fa66fdcec4874397b07e839f1037ecca to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/fa66fdcec4874397b07e839f1037ecca 2024-11-17T08:16:47,614 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/26b9b31c04564f7aaba4715a792b3e2c to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/26b9b31c04564f7aaba4715a792b3e2c 2024-11-17T08:16:47,614 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/a4e8ce6309bd4bd09daa4a8e6dec2ef7 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/a4e8ce6309bd4bd09daa4a8e6dec2ef7 2024-11-17T08:16:47,615 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/70d3e844d7d1432b8afa91b0fa428e3b to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/70d3e844d7d1432b8afa91b0fa428e3b 2024-11-17T08:16:47,616 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/2055e66129ad4be9a3b43582f6c571f1, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/1436fc9320964b87b05ba3badc6c137a, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/d83fda56faf449698065fdc39efe2bd7, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/b36c2e7b3f20448faea602af57198f6b, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/685726a7a8d349ccb24ed373c9099649, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/c4b35217441c4ce4a6471c7f8ea176e9, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/b6579b248233456f9c677f4b0c7ab378, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/de202e4c093c443da3df448adf077de9, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/273f89913f534258ace2097b0e81e67b, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/6b1c65eeb8ab47c08ce994fb0c40a74b, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/25180f4dedd5491fa3ebbaeca6fe86d6, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/4ba557d762aa47ed8e84e0567616cce8, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/4abc988953c948a08319e4515f6dd030, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/7b08bddd166f42c0957b727aaab410e7, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/7b423f312a864dc1b0edd856138b36de, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/ae7294bb9b8543f0899ee1a3c2080be4, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/7d98e73a50ec450d959090288ac58c04, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/9c7ec94cf2e244b3b928fd9fef0d7b17, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/07c47968caa247d09b6b00b7ab58be28, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/a2f117896f914daf8949f3286b57d3a6, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/e86bdbc5fbe148efb88ec4f3cd79ca02, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/c36a96e06c15431aac3a3785772e99fd, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/39e0adf048914e5aa99816ab90a4daea] to archive 2024-11-17T08:16:47,617 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T08:16:47,618 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/2055e66129ad4be9a3b43582f6c571f1 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/2055e66129ad4be9a3b43582f6c571f1 2024-11-17T08:16:47,618 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/1436fc9320964b87b05ba3badc6c137a to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/1436fc9320964b87b05ba3badc6c137a 2024-11-17T08:16:47,619 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/d83fda56faf449698065fdc39efe2bd7 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/d83fda56faf449698065fdc39efe2bd7 2024-11-17T08:16:47,620 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/b36c2e7b3f20448faea602af57198f6b to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/b36c2e7b3f20448faea602af57198f6b 2024-11-17T08:16:47,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/685726a7a8d349ccb24ed373c9099649 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/685726a7a8d349ccb24ed373c9099649 2024-11-17T08:16:47,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/c4b35217441c4ce4a6471c7f8ea176e9 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/c4b35217441c4ce4a6471c7f8ea176e9 2024-11-17T08:16:47,622 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/b6579b248233456f9c677f4b0c7ab378 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/b6579b248233456f9c677f4b0c7ab378 2024-11-17T08:16:47,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/de202e4c093c443da3df448adf077de9 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/de202e4c093c443da3df448adf077de9 2024-11-17T08:16:47,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/273f89913f534258ace2097b0e81e67b to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/273f89913f534258ace2097b0e81e67b 2024-11-17T08:16:47,624 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/6b1c65eeb8ab47c08ce994fb0c40a74b to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/6b1c65eeb8ab47c08ce994fb0c40a74b 2024-11-17T08:16:47,624 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/25180f4dedd5491fa3ebbaeca6fe86d6 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/25180f4dedd5491fa3ebbaeca6fe86d6 2024-11-17T08:16:47,625 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/4ba557d762aa47ed8e84e0567616cce8 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/4ba557d762aa47ed8e84e0567616cce8 2024-11-17T08:16:47,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/4abc988953c948a08319e4515f6dd030 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/4abc988953c948a08319e4515f6dd030 2024-11-17T08:16:47,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/7b08bddd166f42c0957b727aaab410e7 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/7b08bddd166f42c0957b727aaab410e7 2024-11-17T08:16:47,627 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/7b423f312a864dc1b0edd856138b36de to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/7b423f312a864dc1b0edd856138b36de 2024-11-17T08:16:47,627 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/ae7294bb9b8543f0899ee1a3c2080be4 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/ae7294bb9b8543f0899ee1a3c2080be4 2024-11-17T08:16:47,628 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/7d98e73a50ec450d959090288ac58c04 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/7d98e73a50ec450d959090288ac58c04 2024-11-17T08:16:47,629 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/9c7ec94cf2e244b3b928fd9fef0d7b17 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/9c7ec94cf2e244b3b928fd9fef0d7b17 2024-11-17T08:16:47,630 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/07c47968caa247d09b6b00b7ab58be28 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/07c47968caa247d09b6b00b7ab58be28 2024-11-17T08:16:47,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/a2f117896f914daf8949f3286b57d3a6 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/a2f117896f914daf8949f3286b57d3a6 2024-11-17T08:16:47,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/e86bdbc5fbe148efb88ec4f3cd79ca02 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/e86bdbc5fbe148efb88ec4f3cd79ca02 2024-11-17T08:16:47,632 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/c36a96e06c15431aac3a3785772e99fd to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/c36a96e06c15431aac3a3785772e99fd 2024-11-17T08:16:47,632 DEBUG [StoreCloser-TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/39e0adf048914e5aa99816ab90a4daea to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/39e0adf048914e5aa99816ab90a4daea 2024-11-17T08:16:47,636 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/recovered.edits/536.seqid, newMaxSeqId=536, maxSeqId=1 2024-11-17T08:16:47,636 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. 2024-11-17T08:16:47,636 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for c34c3478acc81ba3f3c8f5caec36c13b: Waiting for close lock at 1731831406317Running coprocessor pre-close hooks at 1731831406317Disabling compacts and flushes for region at 1731831406317Disabling writes for close at 1731831406317Obtaining lock to block concurrent updates at 1731831406317Preparing flush snapshotting stores in c34c3478acc81ba3f3c8f5caec36c13b at 1731831406317Finished memstore snapshotting TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=21120, getOffHeapSize=0, getCellsCount=150 at 1731831406317Flushing stores of TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b. at 1731831406318 (+1 ms)Flushing c34c3478acc81ba3f3c8f5caec36c13b/A: creating writer at 1731831406318Flushing c34c3478acc81ba3f3c8f5caec36c13b/A: appending metadata at 1731831406320 (+2 ms)Flushing c34c3478acc81ba3f3c8f5caec36c13b/A: closing flushed file at 1731831406320Flushing c34c3478acc81ba3f3c8f5caec36c13b/B: creating writer at 1731831406728 (+408 ms)Flushing c34c3478acc81ba3f3c8f5caec36c13b/B: appending metadata at 1731831406730 (+2 ms)Flushing c34c3478acc81ba3f3c8f5caec36c13b/B: closing flushed file at 1731831406730Flushing c34c3478acc81ba3f3c8f5caec36c13b/C: creating writer at 1731831407138 (+408 ms)Flushing c34c3478acc81ba3f3c8f5caec36c13b/C: appending metadata at 1731831407141 (+3 ms)Flushing c34c3478acc81ba3f3c8f5caec36c13b/C: closing flushed file at 1731831407141Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@363595da: reopening flushed file at 1731831407550 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55870613: reopening flushed file at 1731831407555 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@621ebc1f: reopening flushed file at 1731831407560 (+5 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for c34c3478acc81ba3f3c8f5caec36c13b in 1248ms, sequenceid=533, compaction requested=true at 1731831407565 (+5 ms)Writing region close event to WAL at 1731831407634 (+69 ms)Running coprocessor post-close hooks at 1731831407636 (+2 ms)Closed at 1731831407636 2024-11-17T08:16:47,638 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed c34c3478acc81ba3f3c8f5caec36c13b 2024-11-17T08:16:47,638 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=c34c3478acc81ba3f3c8f5caec36c13b, regionState=CLOSED 2024-11-17T08:16:47,639 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 because future has completed 2024-11-17T08:16:47,642 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-17T08:16:47,642 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure c34c3478acc81ba3f3c8f5caec36c13b, server=e3d0adb87c01,40631,1731831247805 in 1.4750 sec 2024-11-17T08:16:47,643 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-17T08:16:47,643 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c34c3478acc81ba3f3c8f5caec36c13b, UNASSIGN in 1.4800 sec 2024-11-17T08:16:47,645 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-17T08:16:47,645 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4820 sec 2024-11-17T08:16:47,646 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731831407645"}]},"ts":"1731831407645"} 2024-11-17T08:16:47,647 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T08:16:47,647 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T08:16:47,648 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4900 sec 2024-11-17T08:16:48,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-17T08:16:48,305 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:16:48,306 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-17T08:16:48,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T08:16:48,311 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T08:16:48,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-17T08:16:48,313 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=145, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T08:16:48,319 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b 2024-11-17T08:16:48,322 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A, FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B, FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C, FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/recovered.edits] 2024-11-17T08:16:48,325 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/28fb9839bc2c4fc78541ac3bd4cb6bc8 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/28fb9839bc2c4fc78541ac3bd4cb6bc8 2024-11-17T08:16:48,326 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/4590002d152e4435a2ab3cd8dfae52cc to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/4590002d152e4435a2ab3cd8dfae52cc 2024-11-17T08:16:48,327 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/b628f058596a48b19ae59fa6e0ac4a2b to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/b628f058596a48b19ae59fa6e0ac4a2b 2024-11-17T08:16:48,328 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/c0efbeeeb10b4ac7b20265c21ba00136 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/A/c0efbeeeb10b4ac7b20265c21ba00136 2024-11-17T08:16:48,330 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/2353f0ee151843189d84468ef41b4453 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/2353f0ee151843189d84468ef41b4453 2024-11-17T08:16:48,331 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/d0ccdbccd42943d28f321c37512fdc80 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/d0ccdbccd42943d28f321c37512fdc80 2024-11-17T08:16:48,333 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/d7ee844507a74b43928da77edf429b81 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/d7ee844507a74b43928da77edf429b81 2024-11-17T08:16:48,334 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/ef9fbcd634864388adba1c628d2a2982 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/B/ef9fbcd634864388adba1c628d2a2982 2024-11-17T08:16:48,337 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/54a7609160134a5fa2180ca280ac950a to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/54a7609160134a5fa2180ca280ac950a 2024-11-17T08:16:48,339 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/65963cf9dfd44198ba0057573d58e09a to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/65963cf9dfd44198ba0057573d58e09a 2024-11-17T08:16:48,340 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/9fac5f42dcc1469684233c1b8cfb8893 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/9fac5f42dcc1469684233c1b8cfb8893 2024-11-17T08:16:48,342 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/ebcc6fb9d02445dc963bad05ae48eb58 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/C/ebcc6fb9d02445dc963bad05ae48eb58 2024-11-17T08:16:48,345 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/recovered.edits/536.seqid to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b/recovered.edits/536.seqid 2024-11-17T08:16:48,346 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/c34c3478acc81ba3f3c8f5caec36c13b 2024-11-17T08:16:48,346 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T08:16:48,348 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=145, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T08:16:48,351 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T08:16:48,354 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T08:16:48,355 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=145, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T08:16:48,355 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T08:16:48,356 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731831408355"}]},"ts":"9223372036854775807"} 2024-11-17T08:16:48,358 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T08:16:48,358 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c34c3478acc81ba3f3c8f5caec36c13b, NAME => 'TestAcidGuarantees,,1731831379012.c34c3478acc81ba3f3c8f5caec36c13b.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T08:16:48,358 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T08:16:48,358 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731831408358"}]},"ts":"9223372036854775807"} 2024-11-17T08:16:48,360 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T08:16:48,361 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=145, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T08:16:48,363 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 55 msec 2024-11-17T08:16:48,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-17T08:16:48,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T08:16:48,426 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:16:48,433 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 250), OpenFileDescriptor=445 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=355 (was 366), ProcessCount=11 (was 11), AvailableMemoryMB=2919 (was 2059) - AvailableMemoryMB LEAK? - 2024-11-17T08:16:48,440 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=355, ProcessCount=11, AvailableMemoryMB=2919 2024-11-17T08:16:48,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T08:16:48,441 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T08:16:48,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T08:16:48,443 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T08:16:48,443 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:48,443 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 146 2024-11-17T08:16:48,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-17T08:16:48,444 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T08:16:48,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742347_1523 (size=1061) 2024-11-17T08:16:48,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-17T08:16:48,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-17T08:16:48,850 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85 2024-11-17T08:16:48,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742348_1524 (size=53) 2024-11-17T08:16:49,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-17T08:16:49,255 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T08:16:49,255 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 68e3358487792e1f6c40accb55544a0f, disabling compactions & flushes 2024-11-17T08:16:49,255 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:49,255 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:49,256 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. after waiting 0 ms 2024-11-17T08:16:49,256 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:49,256 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:49,256 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 68e3358487792e1f6c40accb55544a0f: Waiting for close lock at 1731831409255Disabling compacts and flushes for region at 1731831409255Disabling writes for close at 1731831409256 (+1 ms)Writing region close event to WAL at 1731831409256Closed at 1731831409256 2024-11-17T08:16:49,257 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T08:16:49,257 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731831409257"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731831409257"}]},"ts":"1731831409257"} 2024-11-17T08:16:49,259 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T08:16:49,260 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T08:16:49,260 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731831409260"}]},"ts":"1731831409260"} 2024-11-17T08:16:49,262 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T08:16:49,262 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=68e3358487792e1f6c40accb55544a0f, ASSIGN}] 2024-11-17T08:16:49,263 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=68e3358487792e1f6c40accb55544a0f, ASSIGN 2024-11-17T08:16:49,264 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=68e3358487792e1f6c40accb55544a0f, ASSIGN; state=OFFLINE, location=e3d0adb87c01,40631,1731831247805; forceNewPlan=false, retain=false 2024-11-17T08:16:49,415 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=68e3358487792e1f6c40accb55544a0f, regionState=OPENING, regionLocation=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:49,416 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=68e3358487792e1f6c40accb55544a0f, ASSIGN because future has completed 2024-11-17T08:16:49,416 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805}] 2024-11-17T08:16:49,570 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:49,570 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7752): Opening region: {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} 2024-11-17T08:16:49,571 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:16:49,571 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T08:16:49,571 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7794): checking encryption for 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:16:49,571 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7797): checking classloading for 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:16:49,572 INFO [StoreOpener-68e3358487792e1f6c40accb55544a0f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:16:49,573 INFO [StoreOpener-68e3358487792e1f6c40accb55544a0f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T08:16:49,573 INFO [StoreOpener-68e3358487792e1f6c40accb55544a0f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 68e3358487792e1f6c40accb55544a0f columnFamilyName A 2024-11-17T08:16:49,573 DEBUG [StoreOpener-68e3358487792e1f6c40accb55544a0f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:49,574 INFO [StoreOpener-68e3358487792e1f6c40accb55544a0f-1 {}] regionserver.HStore(327): Store=68e3358487792e1f6c40accb55544a0f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T08:16:49,574 INFO [StoreOpener-68e3358487792e1f6c40accb55544a0f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:16:49,575 INFO [StoreOpener-68e3358487792e1f6c40accb55544a0f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T08:16:49,575 INFO [StoreOpener-68e3358487792e1f6c40accb55544a0f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 68e3358487792e1f6c40accb55544a0f columnFamilyName B 2024-11-17T08:16:49,575 DEBUG [StoreOpener-68e3358487792e1f6c40accb55544a0f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:49,575 INFO [StoreOpener-68e3358487792e1f6c40accb55544a0f-1 {}] regionserver.HStore(327): Store=68e3358487792e1f6c40accb55544a0f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T08:16:49,575 INFO [StoreOpener-68e3358487792e1f6c40accb55544a0f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:16:49,576 INFO [StoreOpener-68e3358487792e1f6c40accb55544a0f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T08:16:49,576 INFO [StoreOpener-68e3358487792e1f6c40accb55544a0f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 68e3358487792e1f6c40accb55544a0f columnFamilyName C 2024-11-17T08:16:49,576 DEBUG [StoreOpener-68e3358487792e1f6c40accb55544a0f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:49,577 INFO [StoreOpener-68e3358487792e1f6c40accb55544a0f-1 {}] regionserver.HStore(327): Store=68e3358487792e1f6c40accb55544a0f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T08:16:49,577 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:49,577 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1038): replaying wal for 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:16:49,578 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f 2024-11-17T08:16:49,578 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f 2024-11-17T08:16:49,579 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1048): stopping wal replay for 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:16:49,579 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1060): Cleaning up temporary data for 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:16:49,579 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T08:16:49,580 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1093): writing seq id for 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:16:49,582 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T08:16:49,582 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1114): Opened 68e3358487792e1f6c40accb55544a0f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60753315, jitterRate=-0.09470506012439728}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T08:16:49,582 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:16:49,583 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1006): Region open journal for 68e3358487792e1f6c40accb55544a0f: Running coprocessor pre-open hook at 1731831409571Writing region info on filesystem at 1731831409571Initializing all the Stores at 1731831409571Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731831409571Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731831409572 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731831409572Cleaning up temporary data from old regions at 1731831409579 (+7 ms)Running coprocessor post-open hooks at 1731831409582 (+3 ms)Region opened successfully at 1731831409583 (+1 ms) 2024-11-17T08:16:49,584 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., pid=148, masterSystemTime=1731831409568 2024-11-17T08:16:49,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-17T08:16:49,585 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:49,585 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:49,586 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=68e3358487792e1f6c40accb55544a0f, regionState=OPEN, openSeqNum=2, regionLocation=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:49,587 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 because future has completed 2024-11-17T08:16:49,589 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-17T08:16:49,589 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure 68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 in 171 msec 2024-11-17T08:16:49,591 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-17T08:16:49,591 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=68e3358487792e1f6c40accb55544a0f, ASSIGN in 327 msec 2024-11-17T08:16:49,592 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T08:16:49,592 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731831409592"}]},"ts":"1731831409592"} 2024-11-17T08:16:49,593 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T08:16:49,594 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T08:16:49,595 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1530 sec 2024-11-17T08:16:50,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-17T08:16:50,593 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:16:50,595 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@348042de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:50,595 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:16:50,595 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:16:50,595 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:16:50,596 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:16:50,596 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:16:50,596 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ca57fea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:50,596 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:16:50,596 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:16:50,596 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:50,597 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40982, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:16:50,598 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20b44465, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:50,598 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:16:50,599 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:16:50,599 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:16:50,600 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47516, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:16:50,600 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T08:16:50,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is e3d0adb87c01,45089,1731831246985 2024-11-17T08:16:50,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@f81968d 2024-11-17T08:16:50,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T08:16:50,602 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40994, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T08:16:50,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T08:16:50,603 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T08:16:50,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-17T08:16:50,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742349_1525 (size=1097) 2024-11-17T08:16:51,015 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-17T08:16:51,015 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-17T08:16:51,017 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T08:16:51,020 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=68e3358487792e1f6c40accb55544a0f, REOPEN/MOVE}] 2024-11-17T08:16:51,021 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=68e3358487792e1f6c40accb55544a0f, REOPEN/MOVE 2024-11-17T08:16:51,022 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=68e3358487792e1f6c40accb55544a0f, regionState=CLOSING, regionLocation=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:51,024 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=68e3358487792e1f6c40accb55544a0f, REOPEN/MOVE because future has completed 2024-11-17T08:16:51,024 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T08:16:51,024 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure 68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805}] 2024-11-17T08:16:51,177 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(122): Close 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:16:51,177 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T08:16:51,177 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1722): Closing 68e3358487792e1f6c40accb55544a0f, disabling compactions & flushes 2024-11-17T08:16:51,177 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:51,177 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:51,177 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. after waiting 0 ms 2024-11-17T08:16:51,177 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:51,183 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T08:16:51,184 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:51,184 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1676): Region close journal for 68e3358487792e1f6c40accb55544a0f: Waiting for close lock at 1731831411177Running coprocessor pre-close hooks at 1731831411177Disabling compacts and flushes for region at 1731831411177Disabling writes for close at 1731831411177Writing region close event to WAL at 1731831411178 (+1 ms)Running coprocessor post-close hooks at 1731831411184 (+6 ms)Closed at 1731831411184 2024-11-17T08:16:51,184 WARN [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegionServer(3299): Not adding moved region record: 68e3358487792e1f6c40accb55544a0f to self. 2024-11-17T08:16:51,187 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=68e3358487792e1f6c40accb55544a0f, regionState=CLOSED 2024-11-17T08:16:51,188 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(157): Closed 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:16:51,189 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure 68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 because future has completed 2024-11-17T08:16:51,192 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-17T08:16:51,192 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; CloseRegionProcedure 68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 in 166 msec 2024-11-17T08:16:51,193 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=68e3358487792e1f6c40accb55544a0f, REOPEN/MOVE; state=CLOSED, location=e3d0adb87c01,40631,1731831247805; forceNewPlan=false, retain=true 2024-11-17T08:16:51,344 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=68e3358487792e1f6c40accb55544a0f, regionState=OPENING, regionLocation=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:51,345 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=68e3358487792e1f6c40accb55544a0f, REOPEN/MOVE because future has completed 2024-11-17T08:16:51,346 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure 68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805}] 2024-11-17T08:16:51,499 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:51,500 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7752): Opening region: {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} 2024-11-17T08:16:51,500 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:16:51,500 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T08:16:51,500 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7794): checking encryption for 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:16:51,500 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7797): checking classloading for 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:16:51,501 INFO [StoreOpener-68e3358487792e1f6c40accb55544a0f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:16:51,502 INFO [StoreOpener-68e3358487792e1f6c40accb55544a0f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T08:16:51,502 INFO [StoreOpener-68e3358487792e1f6c40accb55544a0f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 68e3358487792e1f6c40accb55544a0f columnFamilyName A 2024-11-17T08:16:51,503 DEBUG [StoreOpener-68e3358487792e1f6c40accb55544a0f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:51,503 INFO [StoreOpener-68e3358487792e1f6c40accb55544a0f-1 {}] regionserver.HStore(327): Store=68e3358487792e1f6c40accb55544a0f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T08:16:51,503 INFO [StoreOpener-68e3358487792e1f6c40accb55544a0f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:16:51,504 INFO [StoreOpener-68e3358487792e1f6c40accb55544a0f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T08:16:51,504 INFO [StoreOpener-68e3358487792e1f6c40accb55544a0f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 68e3358487792e1f6c40accb55544a0f columnFamilyName B 2024-11-17T08:16:51,504 DEBUG [StoreOpener-68e3358487792e1f6c40accb55544a0f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:51,504 INFO [StoreOpener-68e3358487792e1f6c40accb55544a0f-1 {}] regionserver.HStore(327): Store=68e3358487792e1f6c40accb55544a0f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T08:16:51,504 INFO [StoreOpener-68e3358487792e1f6c40accb55544a0f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:16:51,505 INFO [StoreOpener-68e3358487792e1f6c40accb55544a0f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T08:16:51,505 INFO [StoreOpener-68e3358487792e1f6c40accb55544a0f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 68e3358487792e1f6c40accb55544a0f columnFamilyName C 2024-11-17T08:16:51,505 DEBUG [StoreOpener-68e3358487792e1f6c40accb55544a0f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:51,505 INFO [StoreOpener-68e3358487792e1f6c40accb55544a0f-1 {}] regionserver.HStore(327): Store=68e3358487792e1f6c40accb55544a0f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T08:16:51,505 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:51,505 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1038): replaying wal for 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:16:51,506 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f 2024-11-17T08:16:51,507 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f 2024-11-17T08:16:51,508 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1048): stopping wal replay for 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:16:51,508 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1060): Cleaning up temporary data for 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:16:51,508 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T08:16:51,509 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1093): writing seq id for 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:16:51,510 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1114): Opened 68e3358487792e1f6c40accb55544a0f; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73365275, jitterRate=0.09322778880596161}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T08:16:51,510 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:16:51,511 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1006): Region open journal for 68e3358487792e1f6c40accb55544a0f: Running coprocessor pre-open hook at 1731831411500Writing region info on filesystem at 1731831411500Initializing all the Stores at 1731831411501 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731831411501Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731831411501Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731831411501Cleaning up temporary data from old regions at 1731831411508 (+7 ms)Running coprocessor post-open hooks at 1731831411510 (+2 ms)Region opened successfully at 1731831411511 (+1 ms) 2024-11-17T08:16:51,511 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., pid=153, masterSystemTime=1731831411497 2024-11-17T08:16:51,513 DEBUG [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:51,513 INFO [RS_OPEN_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:51,514 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=68e3358487792e1f6c40accb55544a0f, regionState=OPEN, openSeqNum=5, regionLocation=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:51,515 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure 68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 because future has completed 2024-11-17T08:16:51,520 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=151 2024-11-17T08:16:51,520 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=151, state=SUCCESS, hasLock=false; OpenRegionProcedure 68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 in 173 msec 2024-11-17T08:16:51,523 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-17T08:16:51,523 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=68e3358487792e1f6c40accb55544a0f, REOPEN/MOVE in 501 msec 2024-11-17T08:16:51,525 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-17T08:16:51,525 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 506 msec 2024-11-17T08:16:51,527 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 922 msec 2024-11-17T08:16:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-17T08:16:51,528 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:16:51,529 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24560e66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:51,529 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:16:51,529 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:16:51,529 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:16:51,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:16:51,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:16:51,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@477f86c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:51,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:16:51,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:16:51,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:51,530 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41010, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:16:51,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@959d669, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:51,531 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e83be8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:51,531 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:16:51,531 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:16:51,532 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:16:51,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:16:51,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:16:51,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f275309, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:51,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:16:51,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:16:51,533 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:51,533 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:16:51,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c469b4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:51,534 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49c8d9ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:51,534 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:16:51,535 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:16:51,535 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:16:51,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:16:51,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:16:51,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27dc2280, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:51,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:16:51,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:16:51,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:51,536 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41048, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:16:51,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@442d0ac9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:51,537 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f0cd5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:51,537 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:16:51,538 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:16:51,538 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:16:51,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:16:51,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:16:51,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@468f499, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:51,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:16:51,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:16:51,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:51,539 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41078, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:16:51,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@751e6fc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:51,541 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41947491, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:51,541 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:16:51,541 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:16:51,541 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:16:51,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:16:51,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:16:51,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ee3aaa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:51,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:16:51,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:16:51,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:51,543 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41096, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:16:51,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42f98f91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:51,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19bc8537, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:51,544 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:16:51,544 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:16:51,544 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:16:51,544 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:16:51,544 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:16:51,544 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@621e2c51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:51,544 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:16:51,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:16:51,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:51,545 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41106, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:16:51,545 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@121a38e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:51,546 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7595d14b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:51,546 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:16:51,546 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:16:51,547 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:16:51,547 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:16:51,547 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:16:51,547 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@284bb1a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:51,547 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:16:51,547 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:16:51,547 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:51,547 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:16:51,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16c8f476, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:51,548 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bf79e45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:51,548 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:16:51,548 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:16:51,549 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:16:51,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:16:51,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:16:51,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59e99c49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:51,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:16:51,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:16:51,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:51,550 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:16:51,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e675585, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:51,551 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d0e9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:51,551 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:16:51,551 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:16:51,552 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:16:51,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:16:51,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:16:51,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@340b51d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:51,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:16:51,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:16:51,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:51,553 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:16:51,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3731212d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:51,554 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f9a2803, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:51,554 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3d0adb87c01,45089,-1 for getting cluster id 2024-11-17T08:16:51,554 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T08:16:51,554 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '15a697f6-d2a3-47b3-8935-91acf4ef5fd5' 2024-11-17T08:16:51,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T08:16:51,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "15a697f6-d2a3-47b3-8935-91acf4ef5fd5" 2024-11-17T08:16:51,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6632c331, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:51,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3d0adb87c01,45089,-1] 2024-11-17T08:16:51,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T08:16:51,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:16:51,555 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41174, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T08:16:51,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af7e1f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T08:16:51,556 DEBUG [Thread-2363 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:16:51,556 DEBUG [Thread-2369 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:16:51,557 DEBUG [Thread-2367 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:16:51,557 DEBUG [Thread-2360 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:16:51,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:16:51,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:16:51,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:16:51,557 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:16:51,557 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:16:51,557 DEBUG [Thread-2352 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:16:51,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:16:51,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:16:51,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:16:51,557 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:16:51,558 DEBUG [Thread-2354 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:16:51,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:16:51,558 DEBUG [Thread-2371 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:16:51,558 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:16:51,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:16:51,559 DEBUG [Thread-2356 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:16:51,559 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:16:51,559 DEBUG [Thread-2358 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:16:51,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:16:51,559 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47558, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:16:51,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:16:51,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:16:51,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:16:51,560 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47582, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:16:51,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:16:51,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:16:51,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:16:51,560 DEBUG [Thread-2365 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T08:16:51,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:16:51,560 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47538, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:16:51,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5] 2024-11-17T08:16:51,560 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:16:51,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5] 2024-11-17T08:16:51,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5] 2024-11-17T08:16:51,561 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3d0adb87c01,40631,1731831247805, seqNum=-1] 2024-11-17T08:16:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-17T08:16:51,561 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T08:16:51,561 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:16:51,561 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:16:51,561 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47604, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:16:51,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5] 2024-11-17T08:16:51,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5] 2024-11-17T08:16:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T08:16:51,563 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T08:16:51,563 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5] 2024-11-17T08:16:51,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5] 2024-11-17T08:16:51,563 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:16:51,564 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5] 2024-11-17T08:16:51,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5] 2024-11-17T08:16:51,565 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:16:51,565 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5] 2024-11-17T08:16:51,565 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:16:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:16:51,568 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 68e3358487792e1f6c40accb55544a0f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T08:16:51,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=A 2024-11-17T08:16:51,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:51,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=B 2024-11-17T08:16:51,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:51,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=C 2024-11-17T08:16:51,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:51,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111722e9f138fdcf4afb8c1c96e0613da950_68e3358487792e1f6c40accb55544a0f is 50, key is test_row_0/A:col10/1731831411567/Put/seqid=0 2024-11-17T08:16:51,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742350_1526 (size=14594) 2024-11-17T08:16:51,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:51,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831471613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831471614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:51,618 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731831411609, max timestamp=1731831411616Num uniques 859; 2024-11-17T08:16:51,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:51,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:51,618 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:51,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:51,618 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T08:16:51,618 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831471615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:51,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:51,618 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:51,618 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731831411609, max timestamp=1731831411616Num uniques 859; 2024-11-17T08:16:51,618 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T08:16:51,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:51,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:51,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:51,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731831411609, max timestamp=1731831411616Num uniques 859; 2024-11-17T08:16:51,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e3d0adb87c01:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T08:16:51,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47604 deadline: 1731831471657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:51,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:51,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47582 deadline: 1731831471658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:51,662 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:51,662 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:51,662 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:51,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:51,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:51,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T08:16:51,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T08:16:51,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:51,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:16:51,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:51,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:51,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:51,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:51,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831471723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:51,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:51,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:51,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:51,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:51,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831471723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:51,726 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:51,726 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:51,726 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:51,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:51,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831471723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:51,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:51,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:51,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:51,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:51,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47604 deadline: 1731831471774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:51,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:51,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47582 deadline: 1731831471774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:51,778 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:51,778 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:51,778 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:51,778 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:51,778 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:51,778 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:51,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T08:16:51,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:51,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:16:51,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:51,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:51,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:51,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:51,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T08:16:51,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:51,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831471933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:51,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:51,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831471933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:51,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:51,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:51,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:51,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:51,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:51,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831471934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:51,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:51,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:51,936 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:51,936 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:51,936 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:51,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:51,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:51,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47582 deadline: 1731831471984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:51,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47604 deadline: 1731831471984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:51,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:51,988 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:51,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:51,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:51,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:51,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:51,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:51,993 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111722e9f138fdcf4afb8c1c96e0613da950_68e3358487792e1f6c40accb55544a0f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111722e9f138fdcf4afb8c1c96e0613da950_68e3358487792e1f6c40accb55544a0f 2024-11-17T08:16:51,994 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/a86829658e9949f28fdb9ddfc090dba7, store: [table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:16:51,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/a86829658e9949f28fdb9ddfc090dba7 is 175, key is test_row_0/A:col10/1731831411567/Put/seqid=0 2024-11-17T08:16:52,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742351_1527 (size=39549) 2024-11-17T08:16:52,002 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/a86829658e9949f28fdb9ddfc090dba7 2024-11-17T08:16:52,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T08:16:52,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:52,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:16:52,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:52,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:52,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:52,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:52,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/d7e369bc33304acf8c131c1a6f26d912 is 50, key is test_row_0/B:col10/1731831411567/Put/seqid=0 2024-11-17T08:16:52,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742352_1528 (size=12001) 2024-11-17T08:16:52,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T08:16:52,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:52,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:16:52,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:52,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:52,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:52,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T08:16:52,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:52,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831472243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:52,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:52,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:52,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:52,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831472243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:52,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:52,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:52,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:52,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:52,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831472243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:52,247 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:52,247 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:52,247 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:52,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47582 deadline: 1731831472294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:52,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47604 deadline: 1731831472294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:52,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:52,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:52,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:52,296 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:52,296 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:52,296 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:52,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T08:16:52,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:52,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:16:52,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:52,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:52,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:52,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:52,438 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/d7e369bc33304acf8c131c1a6f26d912 2024-11-17T08:16:52,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/ba1537e2fb8e422e97d825e9681028b8 is 50, key is test_row_0/C:col10/1731831411567/Put/seqid=0 2024-11-17T08:16:52,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742353_1529 (size=12001) 2024-11-17T08:16:52,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T08:16:52,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:52,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:16:52,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:52,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:52,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:52,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T08:16:52,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:52,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:16:52,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:52,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:52,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:52,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:52,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T08:16:52,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:52,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831472753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:52,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:52,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:52,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:52,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831472753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:52,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:52,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:52,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:52,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:52,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:52,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831472754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:52,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:52,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:52,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:52,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T08:16:52,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:52,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:16:52,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:52,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:52,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:52,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:52,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:52,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47582 deadline: 1731831472803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:52,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:52,806 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:52,806 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:52,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47604 deadline: 1731831472803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:52,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:52,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:52,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:52,862 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/ba1537e2fb8e422e97d825e9681028b8 2024-11-17T08:16:52,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/a86829658e9949f28fdb9ddfc090dba7 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/a86829658e9949f28fdb9ddfc090dba7 2024-11-17T08:16:52,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/a86829658e9949f28fdb9ddfc090dba7, entries=200, sequenceid=14, filesize=38.6 K 2024-11-17T08:16:52,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/d7e369bc33304acf8c131c1a6f26d912 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/d7e369bc33304acf8c131c1a6f26d912 2024-11-17T08:16:52,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/d7e369bc33304acf8c131c1a6f26d912, entries=150, sequenceid=14, filesize=11.7 K 2024-11-17T08:16:52,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/ba1537e2fb8e422e97d825e9681028b8 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/ba1537e2fb8e422e97d825e9681028b8 2024-11-17T08:16:52,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/ba1537e2fb8e422e97d825e9681028b8, entries=150, sequenceid=14, filesize=11.7 K 2024-11-17T08:16:52,874 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 68e3358487792e1f6c40accb55544a0f in 1306ms, sequenceid=14, compaction requested=false 2024-11-17T08:16:52,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 68e3358487792e1f6c40accb55544a0f: 2024-11-17T08:16:52,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T08:16:52,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:52,939 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 68e3358487792e1f6c40accb55544a0f 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-17T08:16:52,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=A 2024-11-17T08:16:52,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:16:52,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=B 2024-11-17T08:16:52,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:16:52,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=C 2024-11-17T08:16:52,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T08:16:52,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411177058e670f23c484e9fb6b8e520ef2c99_68e3358487792e1f6c40accb55544a0f is 50, key is test_row_0/A:col10/1731831411631/Put/seqid=0 2024-11-17T08:16:52,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742354_1530 (size=12154) 2024-11-17T08:16:53,153 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T08:16:53,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:53,361 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411177058e670f23c484e9fb6b8e520ef2c99_68e3358487792e1f6c40accb55544a0f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411177058e670f23c484e9fb6b8e520ef2c99_68e3358487792e1f6c40accb55544a0f 2024-11-17T08:16:53,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/30fff306337d4bb58e5458dc754d5efd, store: [table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:16:53,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/30fff306337d4bb58e5458dc754d5efd is 175, key is test_row_0/A:col10/1731831411631/Put/seqid=0 2024-11-17T08:16:53,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742355_1531 (size=30955) 2024-11-17T08:16:53,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T08:16:53,766 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/30fff306337d4bb58e5458dc754d5efd 2024-11-17T08:16:53,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:16:53,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:16:53,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/4cc7c38f8e9a4092995bdcc0964d216d is 50, key is test_row_0/B:col10/1731831411631/Put/seqid=0 2024-11-17T08:16:53,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742356_1532 (size=12001) 2024-11-17T08:16:53,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:53,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831473777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:53,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:53,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:53,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:53,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:53,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:53,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831473783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:53,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831473783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:53,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:53,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:53,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:53,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:53,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:53,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:53,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:53,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47582 deadline: 1731831473814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:53,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:53,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47604 deadline: 1731831473814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:53,819 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to e3d0adb87c01,40631,1731831247805 for 'test_row_0' in 68e3358487792e1f6c40accb55544a0f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T08:16:53,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:53,819 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to e3d0adb87c01,40631,1731831247805 for 'test_row_2' in 68e3358487792e1f6c40accb55544a0f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T08:16:53,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:53,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:53,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:53,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:53,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:53,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:53,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831473894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:53,897 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:53,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:53,897 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:53,897 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:53,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831473894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:53,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:53,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:53,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:53,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:53,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831473894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:53,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:53,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:53,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:54,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:54,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831474103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:54,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:54,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:54,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831474104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:54,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:54,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:54,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:54,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:54,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831474104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:54,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:54,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:54,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:54,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:54,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:54,176 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/4cc7c38f8e9a4092995bdcc0964d216d 2024-11-17T08:16:54,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/3e7d3c8c3e9a46fbbe2e25070d94cdc5 is 50, key is test_row_0/C:col10/1731831411631/Put/seqid=0 2024-11-17T08:16:54,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742357_1533 (size=12001) 2024-11-17T08:16:54,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:54,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831474413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:54,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:54,415 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:54,415 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:54,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831474413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:54,415 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:54,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:54,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:54,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:54,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:54,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831474413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:54,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:54,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:54,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:54,586 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/3e7d3c8c3e9a46fbbe2e25070d94cdc5 2024-11-17T08:16:54,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/30fff306337d4bb58e5458dc754d5efd as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/30fff306337d4bb58e5458dc754d5efd 2024-11-17T08:16:54,591 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/30fff306337d4bb58e5458dc754d5efd, entries=150, sequenceid=50, filesize=30.2 K 2024-11-17T08:16:54,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/4cc7c38f8e9a4092995bdcc0964d216d as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/4cc7c38f8e9a4092995bdcc0964d216d 2024-11-17T08:16:54,595 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/4cc7c38f8e9a4092995bdcc0964d216d, entries=150, sequenceid=50, filesize=11.7 K 2024-11-17T08:16:54,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/3e7d3c8c3e9a46fbbe2e25070d94cdc5 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/3e7d3c8c3e9a46fbbe2e25070d94cdc5 2024-11-17T08:16:54,599 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/3e7d3c8c3e9a46fbbe2e25070d94cdc5, entries=150, sequenceid=50, filesize=11.7 K 2024-11-17T08:16:54,600 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 68e3358487792e1f6c40accb55544a0f in 1661ms, sequenceid=50, compaction requested=false 2024-11-17T08:16:54,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 68e3358487792e1f6c40accb55544a0f: 2024-11-17T08:16:54,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:54,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-17T08:16:54,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-17T08:16:54,603 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-17T08:16:54,604 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0370 sec 2024-11-17T08:16:54,605 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 3.0470 sec 2024-11-17T08:16:54,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:16:54,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 68e3358487792e1f6c40accb55544a0f 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-17T08:16:54,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=A 2024-11-17T08:16:54,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:54,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=B 2024-11-17T08:16:54,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:54,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=C 2024-11-17T08:16:54,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:54,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a72d0d4d266c4e5093731bf19c50335d_68e3358487792e1f6c40accb55544a0f is 50, key is test_row_0/A:col10/1731831413776/Put/seqid=0 2024-11-17T08:16:54,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742358_1534 (size=14594) 2024-11-17T08:16:54,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:54,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831474964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:54,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:54,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:54,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:54,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:54,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831474968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:54,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:54,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831474969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:54,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:54,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:54,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:54,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:54,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:54,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:55,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:55,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831475073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:55,075 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:55,075 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:55,075 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:55,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831475084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:55,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:55,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831475084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:55,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:55,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:55,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:55,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:55,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:55,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:55,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831475283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:55,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:55,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:55,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:55,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:55,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831475293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:55,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:55,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:55,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:55,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831475293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:55,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:55,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:55,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:55,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:55,368 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:55,371 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a72d0d4d266c4e5093731bf19c50335d_68e3358487792e1f6c40accb55544a0f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a72d0d4d266c4e5093731bf19c50335d_68e3358487792e1f6c40accb55544a0f 2024-11-17T08:16:55,372 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/b8785aeece1b4a4580ae7d2467ff77cf, store: [table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:16:55,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/b8785aeece1b4a4580ae7d2467ff77cf is 175, key is test_row_0/A:col10/1731831413776/Put/seqid=0 2024-11-17T08:16:55,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742359_1535 (size=39549) 2024-11-17T08:16:55,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:55,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831475594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:55,599 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:55,599 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:55,599 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:55,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:55,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831475604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:55,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:55,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:55,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:55,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:55,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831475604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:55,607 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:55,607 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:55,607 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T08:16:55,724 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:16:55,725 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:16:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-17T08:16:55,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T08:16:55,726 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:16:55,727 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:16:55,727 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:16:55,775 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/b8785aeece1b4a4580ae7d2467ff77cf 2024-11-17T08:16:55,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/fdcecd8e2a3442a0b85bf2153bbe39b4 is 50, key is test_row_0/B:col10/1731831413776/Put/seqid=0 2024-11-17T08:16:55,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742360_1536 (size=12001) 2024-11-17T08:16:55,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47582 deadline: 1731831475824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:55,827 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to e3d0adb87c01,40631,1731831247805 for 'test_row_0' in 68e3358487792e1f6c40accb55544a0f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T08:16:55,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:55,827 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:55,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:55,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T08:16:55,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47604 deadline: 1731831475834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:55,838 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to e3d0adb87c01,40631,1731831247805 for 'test_row_2' in 68e3358487792e1f6c40accb55544a0f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T08:16:55,838 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:55,838 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:55,838 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:55,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-17T08:16:55,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:55,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:16:55,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:55,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:55,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:55,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:56,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-17T08:16:56,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:56,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:16:56,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:56,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:56,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:56,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:56,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T08:16:56,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:56,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831476103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:56,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:56,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:56,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:56,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:56,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831476114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:56,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:56,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831476114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:56,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:56,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:56,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:56,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:56,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:56,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:56,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/fdcecd8e2a3442a0b85bf2153bbe39b4 2024-11-17T08:16:56,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-17T08:16:56,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:56,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:16:56,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:56,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:56,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:56,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:56,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/6e79efe437e84845bf618ff7e3c3a3d9 is 50, key is test_row_0/C:col10/1731831413776/Put/seqid=0 2024-11-17T08:16:56,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742361_1537 (size=12001) 2024-11-17T08:16:56,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-17T08:16:56,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:56,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:16:56,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:56,338 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:56,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:56,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:56,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T08:16:56,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-17T08:16:56,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:56,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:16:56,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:56,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:56,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:56,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:56,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/6e79efe437e84845bf618ff7e3c3a3d9 2024-11-17T08:16:56,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/b8785aeece1b4a4580ae7d2467ff77cf as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/b8785aeece1b4a4580ae7d2467ff77cf 2024-11-17T08:16:56,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/b8785aeece1b4a4580ae7d2467ff77cf, entries=200, sequenceid=62, filesize=38.6 K 2024-11-17T08:16:56,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/fdcecd8e2a3442a0b85bf2153bbe39b4 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/fdcecd8e2a3442a0b85bf2153bbe39b4 2024-11-17T08:16:56,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/fdcecd8e2a3442a0b85bf2153bbe39b4, entries=150, sequenceid=62, filesize=11.7 K 2024-11-17T08:16:56,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/6e79efe437e84845bf618ff7e3c3a3d9 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/6e79efe437e84845bf618ff7e3c3a3d9 2024-11-17T08:16:56,618 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/6e79efe437e84845bf618ff7e3c3a3d9, entries=150, sequenceid=62, filesize=11.7 K 2024-11-17T08:16:56,619 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 68e3358487792e1f6c40accb55544a0f in 1692ms, sequenceid=62, compaction requested=true 2024-11-17T08:16:56,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 68e3358487792e1f6c40accb55544a0f: 2024-11-17T08:16:56,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68e3358487792e1f6c40accb55544a0f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:16:56,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:56,619 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:16:56,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68e3358487792e1f6c40accb55544a0f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:16:56,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:56,619 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:16:56,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68e3358487792e1f6c40accb55544a0f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:16:56,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:16:56,620 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:16:56,620 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): 68e3358487792e1f6c40accb55544a0f/A is initiating minor compaction (all files) 2024-11-17T08:16:56,620 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68e3358487792e1f6c40accb55544a0f/A in TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:56,620 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/a86829658e9949f28fdb9ddfc090dba7, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/30fff306337d4bb58e5458dc754d5efd, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/b8785aeece1b4a4580ae7d2467ff77cf] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp, totalSize=107.5 K 2024-11-17T08:16:56,620 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:56,620 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. files: [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/a86829658e9949f28fdb9ddfc090dba7, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/30fff306337d4bb58e5458dc754d5efd, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/b8785aeece1b4a4580ae7d2467ff77cf] 2024-11-17T08:16:56,620 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:16:56,620 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): 68e3358487792e1f6c40accb55544a0f/B is initiating minor compaction (all files) 2024-11-17T08:16:56,620 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting a86829658e9949f28fdb9ddfc090dba7, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731831411564 2024-11-17T08:16:56,620 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68e3358487792e1f6c40accb55544a0f/B in TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:56,620 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/d7e369bc33304acf8c131c1a6f26d912, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/4cc7c38f8e9a4092995bdcc0964d216d, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/fdcecd8e2a3442a0b85bf2153bbe39b4] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp, totalSize=35.2 K 2024-11-17T08:16:56,620 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30fff306337d4bb58e5458dc754d5efd, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731831411631 2024-11-17T08:16:56,621 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting d7e369bc33304acf8c131c1a6f26d912, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731831411564 2024-11-17T08:16:56,621 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8785aeece1b4a4580ae7d2467ff77cf, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731831413776 2024-11-17T08:16:56,621 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cc7c38f8e9a4092995bdcc0964d216d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731831411631 2024-11-17T08:16:56,622 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting fdcecd8e2a3442a0b85bf2153bbe39b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731831413776 2024-11-17T08:16:56,632 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68e3358487792e1f6c40accb55544a0f#B#compaction#456 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:56,632 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/0f3de109dcea444dbaa0504eaa55fbc4 is 50, key is test_row_0/B:col10/1731831413776/Put/seqid=0 2024-11-17T08:16:56,633 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:16:56,635 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111705f8e4c814da4fe0a3daa82d2636f7ec_68e3358487792e1f6c40accb55544a0f store=[table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:16:56,638 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111705f8e4c814da4fe0a3daa82d2636f7ec_68e3358487792e1f6c40accb55544a0f, store=[table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:16:56,638 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111705f8e4c814da4fe0a3daa82d2636f7ec_68e3358487792e1f6c40accb55544a0f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:16:56,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742362_1538 (size=12104) 2024-11-17T08:16:56,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-17T08:16:56,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:56,644 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 68e3358487792e1f6c40accb55544a0f 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-17T08:16:56,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=A 2024-11-17T08:16:56,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:56,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=B 2024-11-17T08:16:56,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:56,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=C 2024-11-17T08:16:56,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:56,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742363_1539 (size=4469) 2024-11-17T08:16:56,651 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68e3358487792e1f6c40accb55544a0f#A#compaction#457 average throughput is 1.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:56,652 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/defe01e0d1294dfa94ff7d22514ab2a3 is 175, key is test_row_0/A:col10/1731831413776/Put/seqid=0 2024-11-17T08:16:56,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411171ecbca7b1c604396a888eff77a674237_68e3358487792e1f6c40accb55544a0f is 50, key is test_row_0/A:col10/1731831414967/Put/seqid=0 2024-11-17T08:16:56,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742364_1540 (size=31058) 2024-11-17T08:16:56,672 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/defe01e0d1294dfa94ff7d22514ab2a3 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/defe01e0d1294dfa94ff7d22514ab2a3 2024-11-17T08:16:56,679 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68e3358487792e1f6c40accb55544a0f/A of 68e3358487792e1f6c40accb55544a0f into defe01e0d1294dfa94ff7d22514ab2a3(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:56,679 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68e3358487792e1f6c40accb55544a0f: 2024-11-17T08:16:56,679 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., storeName=68e3358487792e1f6c40accb55544a0f/A, priority=13, startTime=1731831416619; duration=0sec 2024-11-17T08:16:56,679 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:16:56,679 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68e3358487792e1f6c40accb55544a0f:A 2024-11-17T08:16:56,680 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:16:56,681 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:16:56,681 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): 68e3358487792e1f6c40accb55544a0f/C is initiating minor compaction (all files) 2024-11-17T08:16:56,681 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68e3358487792e1f6c40accb55544a0f/C in TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:56,681 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/ba1537e2fb8e422e97d825e9681028b8, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/3e7d3c8c3e9a46fbbe2e25070d94cdc5, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/6e79efe437e84845bf618ff7e3c3a3d9] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp, totalSize=35.2 K 2024-11-17T08:16:56,681 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba1537e2fb8e422e97d825e9681028b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731831411564 2024-11-17T08:16:56,682 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e7d3c8c3e9a46fbbe2e25070d94cdc5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731831411631 2024-11-17T08:16:56,683 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e79efe437e84845bf618ff7e3c3a3d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731831413776 2024-11-17T08:16:56,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742365_1541 (size=12154) 2024-11-17T08:16:56,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:56,703 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411171ecbca7b1c604396a888eff77a674237_68e3358487792e1f6c40accb55544a0f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411171ecbca7b1c604396a888eff77a674237_68e3358487792e1f6c40accb55544a0f 2024-11-17T08:16:56,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/1b5bafb627ca4344981204f6a90a7c73, store: [table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:16:56,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/1b5bafb627ca4344981204f6a90a7c73 is 175, key is test_row_0/A:col10/1731831414967/Put/seqid=0 2024-11-17T08:16:56,714 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68e3358487792e1f6c40accb55544a0f#C#compaction#459 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:56,715 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/122ba7313e664c30bf9e7dea0ca40805 is 50, key is test_row_0/C:col10/1731831413776/Put/seqid=0 2024-11-17T08:16:56,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742366_1542 (size=30955) 2024-11-17T08:16:56,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742367_1543 (size=12104) 2024-11-17T08:16:56,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T08:16:57,045 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/0f3de109dcea444dbaa0504eaa55fbc4 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/0f3de109dcea444dbaa0504eaa55fbc4 2024-11-17T08:16:57,048 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68e3358487792e1f6c40accb55544a0f/B of 68e3358487792e1f6c40accb55544a0f into 0f3de109dcea444dbaa0504eaa55fbc4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:57,048 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68e3358487792e1f6c40accb55544a0f: 2024-11-17T08:16:57,048 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., storeName=68e3358487792e1f6c40accb55544a0f/B, priority=13, startTime=1731831416619; duration=0sec 2024-11-17T08:16:57,048 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:57,048 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68e3358487792e1f6c40accb55544a0f:B 2024-11-17T08:16:57,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:16:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:16:57,133 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/1b5bafb627ca4344981204f6a90a7c73 2024-11-17T08:16:57,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831477131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:57,136 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:57,136 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:57,136 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:57,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831477132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:57,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831477132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:57,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:57,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:57,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:57,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/f7594801fa044550a0f0a3e98139836e is 50, key is test_row_0/B:col10/1731831414967/Put/seqid=0 2024-11-17T08:16:57,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742368_1544 (size=12001) 2024-11-17T08:16:57,143 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/f7594801fa044550a0f0a3e98139836e 2024-11-17T08:16:57,146 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/122ba7313e664c30bf9e7dea0ca40805 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/122ba7313e664c30bf9e7dea0ca40805 2024-11-17T08:16:57,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:57,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:57,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:57,160 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68e3358487792e1f6c40accb55544a0f/C of 68e3358487792e1f6c40accb55544a0f into 122ba7313e664c30bf9e7dea0ca40805(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:57,160 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68e3358487792e1f6c40accb55544a0f: 2024-11-17T08:16:57,160 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., storeName=68e3358487792e1f6c40accb55544a0f/C, priority=13, startTime=1731831416619; duration=0sec 2024-11-17T08:16:57,160 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:57,160 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68e3358487792e1f6c40accb55544a0f:C 2024-11-17T08:16:57,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/8ff196210b754475b4245134d5c17271 is 50, key is test_row_0/C:col10/1731831414967/Put/seqid=0 2024-11-17T08:16:57,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742369_1545 (size=12001) 2024-11-17T08:16:57,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:57,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831477244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:57,246 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:57,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:57,246 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:57,246 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:57,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831477244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:57,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:57,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:57,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:57,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:57,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831477254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:57,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:57,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:57,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:57,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:57,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831477454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:57,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831477454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:57,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:57,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831477463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:57,582 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/8ff196210b754475b4245134d5c17271 2024-11-17T08:16:57,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/1b5bafb627ca4344981204f6a90a7c73 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/1b5bafb627ca4344981204f6a90a7c73 2024-11-17T08:16:57,588 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/1b5bafb627ca4344981204f6a90a7c73, entries=150, sequenceid=82, filesize=30.2 K 2024-11-17T08:16:57,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/f7594801fa044550a0f0a3e98139836e as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/f7594801fa044550a0f0a3e98139836e 2024-11-17T08:16:57,593 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/f7594801fa044550a0f0a3e98139836e, entries=150, sequenceid=82, filesize=11.7 K 2024-11-17T08:16:57,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/8ff196210b754475b4245134d5c17271 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/8ff196210b754475b4245134d5c17271 2024-11-17T08:16:57,596 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/8ff196210b754475b4245134d5c17271, entries=150, sequenceid=82, filesize=11.7 K 2024-11-17T08:16:57,597 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 68e3358487792e1f6c40accb55544a0f in 953ms, sequenceid=82, compaction requested=false 2024-11-17T08:16:57,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 68e3358487792e1f6c40accb55544a0f: 2024-11-17T08:16:57,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:57,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-17T08:16:57,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-17T08:16:57,600 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-17T08:16:57,600 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8710 sec 2024-11-17T08:16:57,601 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.8750 sec 2024-11-17T08:16:57,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:16:57,766 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 68e3358487792e1f6c40accb55544a0f 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-17T08:16:57,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=A 2024-11-17T08:16:57,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:57,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=B 2024-11-17T08:16:57,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:57,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=C 2024-11-17T08:16:57,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:57,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d1174a2690614965a674460d8ea82767_68e3358487792e1f6c40accb55544a0f is 50, key is test_row_0/A:col10/1731831417766/Put/seqid=0 2024-11-17T08:16:57,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742370_1546 (size=14594) 2024-11-17T08:16:57,780 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:57,783 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d1174a2690614965a674460d8ea82767_68e3358487792e1f6c40accb55544a0f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d1174a2690614965a674460d8ea82767_68e3358487792e1f6c40accb55544a0f 2024-11-17T08:16:57,783 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/eabf0b31fe6442a3bd10dede5fb3a855, store: [table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:16:57,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/eabf0b31fe6442a3bd10dede5fb3a855 is 175, key is test_row_0/A:col10/1731831417766/Put/seqid=0 2024-11-17T08:16:57,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742371_1547 (size=39549) 2024-11-17T08:16:57,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:57,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831477793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:57,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:57,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:57,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:57,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:57,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831477796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:57,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:57,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831477797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:57,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:57,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:57,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:57,802 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:57,802 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:57,802 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:57,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T08:16:57,874 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:16:57,874 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:16:57,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-17T08:16:57,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T08:16:57,877 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:16:57,878 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:16:57,878 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:16:57,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:57,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831477904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:57,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:57,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:57,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:57,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:57,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831477904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:57,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:57,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:57,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:57,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:57,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831477913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:57,915 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:57,915 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:57,915 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:57,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T08:16:58,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-17T08:16:58,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:58,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:16:58,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:58,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:58,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:58,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:58,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:58,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831478114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:58,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:58,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831478114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:58,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:58,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831478123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:58,125 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:58,126 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:58,126 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:58,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-17T08:16:58,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:58,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:16:58,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:58,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:58,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:58,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:58,189 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/eabf0b31fe6442a3bd10dede5fb3a855 2024-11-17T08:16:58,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T08:16:58,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/abd5284b9d93416aa669e7eb84054d2a is 50, key is test_row_0/B:col10/1731831417766/Put/seqid=0 2024-11-17T08:16:58,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742372_1548 (size=12001) 2024-11-17T08:16:58,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-17T08:16:58,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:58,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:16:58,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:58,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:58,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:58,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:58,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:58,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831478423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:58,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:58,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:58,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:58,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:58,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831478423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:58,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:58,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:58,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:58,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:58,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831478434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:58,436 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:58,436 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:58,436 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:58,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-17T08:16:58,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:58,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:16:58,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:58,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:58,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:58,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:16:58,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T08:16:58,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/abd5284b9d93416aa669e7eb84054d2a 2024-11-17T08:16:58,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/b28fe82ee72b40fb9cd15825ea7eb593 is 50, key is test_row_0/C:col10/1731831417766/Put/seqid=0 2024-11-17T08:16:58,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742373_1549 (size=12001) 2024-11-17T08:16:58,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/b28fe82ee72b40fb9cd15825ea7eb593 2024-11-17T08:16:58,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/eabf0b31fe6442a3bd10dede5fb3a855 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/eabf0b31fe6442a3bd10dede5fb3a855 2024-11-17T08:16:58,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/eabf0b31fe6442a3bd10dede5fb3a855, entries=200, sequenceid=99, filesize=38.6 K 2024-11-17T08:16:58,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/abd5284b9d93416aa669e7eb84054d2a as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/abd5284b9d93416aa669e7eb84054d2a 2024-11-17T08:16:58,618 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/abd5284b9d93416aa669e7eb84054d2a, entries=150, sequenceid=99, filesize=11.7 K 2024-11-17T08:16:58,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/b28fe82ee72b40fb9cd15825ea7eb593 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/b28fe82ee72b40fb9cd15825ea7eb593 2024-11-17T08:16:58,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/b28fe82ee72b40fb9cd15825ea7eb593, entries=150, sequenceid=99, filesize=11.7 K 2024-11-17T08:16:58,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 68e3358487792e1f6c40accb55544a0f in 856ms, sequenceid=99, compaction requested=true 2024-11-17T08:16:58,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 68e3358487792e1f6c40accb55544a0f: 2024-11-17T08:16:58,622 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:16:58,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68e3358487792e1f6c40accb55544a0f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:16:58,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:58,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68e3358487792e1f6c40accb55544a0f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:16:58,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:58,622 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:16:58,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68e3358487792e1f6c40accb55544a0f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:16:58,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:16:58,623 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:16:58,623 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): 68e3358487792e1f6c40accb55544a0f/A is initiating minor compaction (all files) 2024-11-17T08:16:58,623 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:16:58,623 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): 68e3358487792e1f6c40accb55544a0f/B is initiating minor compaction (all files) 2024-11-17T08:16:58,623 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68e3358487792e1f6c40accb55544a0f/B in TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:58,623 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/0f3de109dcea444dbaa0504eaa55fbc4, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/f7594801fa044550a0f0a3e98139836e, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/abd5284b9d93416aa669e7eb84054d2a] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp, totalSize=35.3 K 2024-11-17T08:16:58,623 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68e3358487792e1f6c40accb55544a0f/A in TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:58,623 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/defe01e0d1294dfa94ff7d22514ab2a3, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/1b5bafb627ca4344981204f6a90a7c73, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/eabf0b31fe6442a3bd10dede5fb3a855] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp, totalSize=99.2 K 2024-11-17T08:16:58,623 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:58,623 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. files: [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/defe01e0d1294dfa94ff7d22514ab2a3, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/1b5bafb627ca4344981204f6a90a7c73, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/eabf0b31fe6442a3bd10dede5fb3a855] 2024-11-17T08:16:58,624 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f3de109dcea444dbaa0504eaa55fbc4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731831413776 2024-11-17T08:16:58,624 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting defe01e0d1294dfa94ff7d22514ab2a3, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731831413776 2024-11-17T08:16:58,624 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting f7594801fa044550a0f0a3e98139836e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731831414959 2024-11-17T08:16:58,624 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b5bafb627ca4344981204f6a90a7c73, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731831414959 2024-11-17T08:16:58,625 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting abd5284b9d93416aa669e7eb84054d2a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731831417126 2024-11-17T08:16:58,625 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting eabf0b31fe6442a3bd10dede5fb3a855, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731831417126 2024-11-17T08:16:58,629 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:16:58,631 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68e3358487792e1f6c40accb55544a0f#B#compaction#465 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:58,631 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/8ea14f811ce24a759d9016a4c4fcfd26 is 50, key is test_row_0/B:col10/1731831417766/Put/seqid=0 2024-11-17T08:16:58,632 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111733b157f84c48415db6df58eae504b888_68e3358487792e1f6c40accb55544a0f store=[table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:16:58,634 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111733b157f84c48415db6df58eae504b888_68e3358487792e1f6c40accb55544a0f, store=[table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:16:58,634 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111733b157f84c48415db6df58eae504b888_68e3358487792e1f6c40accb55544a0f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:16:58,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-17T08:16:58,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:58,643 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 68e3358487792e1f6c40accb55544a0f 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-17T08:16:58,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=A 2024-11-17T08:16:58,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:58,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=B 2024-11-17T08:16:58,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:58,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=C 2024-11-17T08:16:58,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:16:58,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742375_1551 (size=4469) 2024-11-17T08:16:58,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a78d1dd8cd3e4ba7aaaf25c445726916_68e3358487792e1f6c40accb55544a0f is 50, key is test_row_0/A:col10/1731831417796/Put/seqid=0 2024-11-17T08:16:58,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742374_1550 (size=12207) 2024-11-17T08:16:58,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742376_1552 (size=12154) 2024-11-17T08:16:58,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:16:58,674 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a78d1dd8cd3e4ba7aaaf25c445726916_68e3358487792e1f6c40accb55544a0f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a78d1dd8cd3e4ba7aaaf25c445726916_68e3358487792e1f6c40accb55544a0f 2024-11-17T08:16:58,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/21f7a7294c0c4a2d8a6a642b2db2b40f, store: [table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:16:58,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/21f7a7294c0c4a2d8a6a642b2db2b40f is 175, key is test_row_0/A:col10/1731831417796/Put/seqid=0 2024-11-17T08:16:58,677 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/8ea14f811ce24a759d9016a4c4fcfd26 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/8ea14f811ce24a759d9016a4c4fcfd26 2024-11-17T08:16:58,682 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68e3358487792e1f6c40accb55544a0f/B of 68e3358487792e1f6c40accb55544a0f into 8ea14f811ce24a759d9016a4c4fcfd26(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:58,682 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68e3358487792e1f6c40accb55544a0f: 2024-11-17T08:16:58,682 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., storeName=68e3358487792e1f6c40accb55544a0f/B, priority=13, startTime=1731831418622; duration=0sec 2024-11-17T08:16:58,682 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:16:58,682 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68e3358487792e1f6c40accb55544a0f:B 2024-11-17T08:16:58,682 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:16:58,683 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:16:58,683 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): 68e3358487792e1f6c40accb55544a0f/C is initiating minor compaction (all files) 2024-11-17T08:16:58,683 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68e3358487792e1f6c40accb55544a0f/C in TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:58,683 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/122ba7313e664c30bf9e7dea0ca40805, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/8ff196210b754475b4245134d5c17271, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/b28fe82ee72b40fb9cd15825ea7eb593] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp, totalSize=35.3 K 2024-11-17T08:16:58,683 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 122ba7313e664c30bf9e7dea0ca40805, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731831413776 2024-11-17T08:16:58,684 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ff196210b754475b4245134d5c17271, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731831414959 2024-11-17T08:16:58,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742377_1553 (size=30955) 2024-11-17T08:16:58,684 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting b28fe82ee72b40fb9cd15825ea7eb593, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731831417126 2024-11-17T08:16:58,690 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68e3358487792e1f6c40accb55544a0f#C#compaction#468 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:58,691 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/e2aa165869614511aa94f1bc65d44673 is 50, key is test_row_0/C:col10/1731831417766/Put/seqid=0 2024-11-17T08:16:58,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742378_1554 (size=12207) 2024-11-17T08:16:58,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:16:58,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:16:58,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:58,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831478956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:58,961 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:58,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:58,961 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:58,961 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:58,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831478957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:58,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:58,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:58,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:58,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:58,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831478958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:58,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:58,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:58,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:59,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T08:16:59,058 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68e3358487792e1f6c40accb55544a0f#A#compaction#466 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:16:59,058 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/d7df533ba0f94fb087b5a453efd69e30 is 175, key is test_row_0/A:col10/1731831417766/Put/seqid=0 2024-11-17T08:16:59,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742379_1555 (size=31161) 2024-11-17T08:16:59,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:59,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831479063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:59,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:59,067 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:59,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831479063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:59,067 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:59,067 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:59,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:59,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:59,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:59,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:59,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831479074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:59,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:59,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:59,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:59,084 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/21f7a7294c0c4a2d8a6a642b2db2b40f 2024-11-17T08:16:59,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/c8f626d32b4d4170bc6c06cae715140c is 50, key is test_row_0/B:col10/1731831417796/Put/seqid=0 2024-11-17T08:16:59,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742380_1556 (size=12001) 2024-11-17T08:16:59,102 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/e2aa165869614511aa94f1bc65d44673 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/e2aa165869614511aa94f1bc65d44673 2024-11-17T08:16:59,105 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68e3358487792e1f6c40accb55544a0f/C of 68e3358487792e1f6c40accb55544a0f into e2aa165869614511aa94f1bc65d44673(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:59,105 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68e3358487792e1f6c40accb55544a0f: 2024-11-17T08:16:59,105 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., storeName=68e3358487792e1f6c40accb55544a0f/C, priority=13, startTime=1731831418622; duration=0sec 2024-11-17T08:16:59,105 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:59,105 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68e3358487792e1f6c40accb55544a0f:C 2024-11-17T08:16:59,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:59,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831479273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:59,276 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:59,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:59,276 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:59,276 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:59,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831479273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:59,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:59,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:59,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:59,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:59,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831479283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:59,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:59,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:59,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:59,464 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/d7df533ba0f94fb087b5a453efd69e30 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/d7df533ba0f94fb087b5a453efd69e30 2024-11-17T08:16:59,467 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68e3358487792e1f6c40accb55544a0f/A of 68e3358487792e1f6c40accb55544a0f into d7df533ba0f94fb087b5a453efd69e30(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:16:59,468 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68e3358487792e1f6c40accb55544a0f: 2024-11-17T08:16:59,468 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., storeName=68e3358487792e1f6c40accb55544a0f/A, priority=13, startTime=1731831418622; duration=0sec 2024-11-17T08:16:59,468 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:16:59,468 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68e3358487792e1f6c40accb55544a0f:A 2024-11-17T08:16:59,495 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/c8f626d32b4d4170bc6c06cae715140c 2024-11-17T08:16:59,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/2ce4cf4653be4096afa12a279a5c13ba is 50, key is test_row_0/C:col10/1731831417796/Put/seqid=0 2024-11-17T08:16:59,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742381_1557 (size=12001) 2024-11-17T08:16:59,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:59,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831479584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:59,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:59,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:59,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:59,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831479584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:59,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:59,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:59,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:59,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:59,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:59,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831479594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:59,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:59,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:59,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:59,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:59,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47604 deadline: 1731831479853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:59,855 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to e3d0adb87c01,40631,1731831247805 for 'test_row_2' in 68e3358487792e1f6c40accb55544a0f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T08:16:59,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:59,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:59,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:59,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:16:59,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47582 deadline: 1731831479863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:16:59,866 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to e3d0adb87c01,40631,1731831247805 for 'test_row_0' in 68e3358487792e1f6c40accb55544a0f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T08:16:59,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:59,867 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:16:59,867 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:16:59,905 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/2ce4cf4653be4096afa12a279a5c13ba 2024-11-17T08:16:59,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/21f7a7294c0c4a2d8a6a642b2db2b40f as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/21f7a7294c0c4a2d8a6a642b2db2b40f 2024-11-17T08:16:59,911 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/21f7a7294c0c4a2d8a6a642b2db2b40f, entries=150, sequenceid=117, filesize=30.2 K 2024-11-17T08:16:59,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/c8f626d32b4d4170bc6c06cae715140c as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/c8f626d32b4d4170bc6c06cae715140c 2024-11-17T08:16:59,915 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/c8f626d32b4d4170bc6c06cae715140c, entries=150, sequenceid=117, filesize=11.7 K 2024-11-17T08:16:59,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/2ce4cf4653be4096afa12a279a5c13ba as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/2ce4cf4653be4096afa12a279a5c13ba 2024-11-17T08:16:59,918 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/2ce4cf4653be4096afa12a279a5c13ba, entries=150, sequenceid=117, filesize=11.7 K 2024-11-17T08:16:59,919 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 68e3358487792e1f6c40accb55544a0f in 1277ms, sequenceid=117, compaction requested=false 2024-11-17T08:16:59,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 68e3358487792e1f6c40accb55544a0f: 2024-11-17T08:16:59,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:16:59,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-17T08:16:59,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-17T08:16:59,942 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-17T08:16:59,942 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0610 sec 2024-11-17T08:16:59,944 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.0690 sec 2024-11-17T08:17:00,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T08:17:00,024 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:17:00,024 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:17:00,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-17T08:17:00,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T08:17:00,026 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:17:00,026 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:17:00,026 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:17:00,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:00,097 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 68e3358487792e1f6c40accb55544a0f 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-17T08:17:00,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=A 2024-11-17T08:17:00,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:17:00,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=B 2024-11-17T08:17:00,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:17:00,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=C 2024-11-17T08:17:00,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:17:00,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111730475c9f375041c7a91e32c51c8ce6a7_68e3358487792e1f6c40accb55544a0f is 50, key is test_row_0/A:col10/1731831420096/Put/seqid=0 2024-11-17T08:17:00,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742382_1558 (size=14744) 2024-11-17T08:17:00,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:00,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831480119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:00,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:00,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:00,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:00,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:00,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831480123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:00,129 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:00,129 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:00,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:00,129 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:00,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831480123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:00,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:00,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:00,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:00,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T08:17:00,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T08:17:00,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:00,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:17:00,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:00,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:00,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:00,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:00,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:00,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831480234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:00,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:00,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:00,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831480234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:00,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831480234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:00,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:00,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:00,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:00,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:00,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:00,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:00,239 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:00,239 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:00,239 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:00,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T08:17:00,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:00,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:17:00,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:00,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:00,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:00,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:00,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T08:17:00,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:00,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831480444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:00,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:00,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831480444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:00,448 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:00,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:00,448 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:00,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831480444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:00,448 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:00,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:00,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:00,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:00,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:00,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:00,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:00,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T08:17:00,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:00,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:17:00,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:00,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:00,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:00,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:00,508 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:00,510 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111730475c9f375041c7a91e32c51c8ce6a7_68e3358487792e1f6c40accb55544a0f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111730475c9f375041c7a91e32c51c8ce6a7_68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:00,511 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/5d3c8a3137434ee1b244441ecfc00875, store: [table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:17:00,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/5d3c8a3137434ee1b244441ecfc00875 is 175, key is test_row_0/A:col10/1731831420096/Put/seqid=0 2024-11-17T08:17:00,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742383_1559 (size=39699) 2024-11-17T08:17:00,514 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/5d3c8a3137434ee1b244441ecfc00875 2024-11-17T08:17:00,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/31bd6a59006843f2b89035179444bfa9 is 50, key is test_row_0/B:col10/1731831420096/Put/seqid=0 2024-11-17T08:17:00,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742384_1560 (size=12151) 2024-11-17T08:17:00,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T08:17:00,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:00,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:17:00,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:00,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:00,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:00,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:00,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T08:17:00,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:00,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831480754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:00,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:00,756 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:00,756 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:00,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831480754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:00,756 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:00,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:00,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:00,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:00,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:00,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831480754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:00,758 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:00,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:00,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:00,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T08:17:00,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:00,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:17:00,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:00,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:00,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:00,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:00,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/31bd6a59006843f2b89035179444bfa9 2024-11-17T08:17:00,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/71516fe5b9db45ce8d97648ad1a3646b is 50, key is test_row_0/C:col10/1731831420096/Put/seqid=0 2024-11-17T08:17:00,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T08:17:00,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:00,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:17:00,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:00,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:00,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:00,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:00,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742385_1561 (size=12151) 2024-11-17T08:17:00,952 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/71516fe5b9db45ce8d97648ad1a3646b 2024-11-17T08:17:00,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/5d3c8a3137434ee1b244441ecfc00875 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/5d3c8a3137434ee1b244441ecfc00875 2024-11-17T08:17:00,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/5d3c8a3137434ee1b244441ecfc00875, entries=200, sequenceid=136, filesize=38.8 K 2024-11-17T08:17:00,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/31bd6a59006843f2b89035179444bfa9 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/31bd6a59006843f2b89035179444bfa9 2024-11-17T08:17:00,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/31bd6a59006843f2b89035179444bfa9, entries=150, sequenceid=136, filesize=11.9 K 2024-11-17T08:17:00,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/71516fe5b9db45ce8d97648ad1a3646b as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/71516fe5b9db45ce8d97648ad1a3646b 2024-11-17T08:17:00,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/71516fe5b9db45ce8d97648ad1a3646b, entries=150, sequenceid=136, filesize=11.9 K 2024-11-17T08:17:00,966 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 68e3358487792e1f6c40accb55544a0f in 868ms, sequenceid=136, compaction requested=true 2024-11-17T08:17:00,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 68e3358487792e1f6c40accb55544a0f: 2024-11-17T08:17:00,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68e3358487792e1f6c40accb55544a0f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:17:00,966 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:17:00,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:17:00,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68e3358487792e1f6c40accb55544a0f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:17:00,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:17:00,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68e3358487792e1f6c40accb55544a0f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:17:00,966 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:17:00,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:17:00,966 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101815 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:17:00,966 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:17:00,967 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): 68e3358487792e1f6c40accb55544a0f/A is initiating minor compaction (all files) 2024-11-17T08:17:00,967 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): 68e3358487792e1f6c40accb55544a0f/B is initiating minor compaction (all files) 2024-11-17T08:17:00,967 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68e3358487792e1f6c40accb55544a0f/A in TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:00,967 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68e3358487792e1f6c40accb55544a0f/B in TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:00,967 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/8ea14f811ce24a759d9016a4c4fcfd26, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/c8f626d32b4d4170bc6c06cae715140c, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/31bd6a59006843f2b89035179444bfa9] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp, totalSize=35.5 K 2024-11-17T08:17:00,967 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/d7df533ba0f94fb087b5a453efd69e30, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/21f7a7294c0c4a2d8a6a642b2db2b40f, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/5d3c8a3137434ee1b244441ecfc00875] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp, totalSize=99.4 K 2024-11-17T08:17:00,967 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:00,967 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. files: [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/d7df533ba0f94fb087b5a453efd69e30, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/21f7a7294c0c4a2d8a6a642b2db2b40f, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/5d3c8a3137434ee1b244441ecfc00875] 2024-11-17T08:17:00,967 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ea14f811ce24a759d9016a4c4fcfd26, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731831417126 2024-11-17T08:17:00,967 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7df533ba0f94fb087b5a453efd69e30, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731831417126 2024-11-17T08:17:00,967 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21f7a7294c0c4a2d8a6a642b2db2b40f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731831417778 2024-11-17T08:17:00,967 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting c8f626d32b4d4170bc6c06cae715140c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731831417778 2024-11-17T08:17:00,967 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d3c8a3137434ee1b244441ecfc00875, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731831418948 2024-11-17T08:17:00,968 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 31bd6a59006843f2b89035179444bfa9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731831418957 2024-11-17T08:17:00,973 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:17:00,974 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68e3358487792e1f6c40accb55544a0f#B#compaction#474 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:17:00,974 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/28a5fa720bd0427c82ae245d16d6e93d is 50, key is test_row_0/B:col10/1731831420096/Put/seqid=0 2024-11-17T08:17:00,975 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117bc147a5a84cc4ecb938a271f75093313_68e3358487792e1f6c40accb55544a0f store=[table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:17:00,977 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117bc147a5a84cc4ecb938a271f75093313_68e3358487792e1f6c40accb55544a0f, store=[table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:17:00,977 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117bc147a5a84cc4ecb938a271f75093313_68e3358487792e1f6c40accb55544a0f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:17:00,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742386_1562 (size=12459) 2024-11-17T08:17:00,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742387_1563 (size=4469) 2024-11-17T08:17:01,000 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68e3358487792e1f6c40accb55544a0f#A#compaction#475 average throughput is 0.90 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:17:01,001 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/47e33686176f490eaf818df9900cabba is 175, key is test_row_0/A:col10/1731831420096/Put/seqid=0 2024-11-17T08:17:01,006 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/28a5fa720bd0427c82ae245d16d6e93d as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/28a5fa720bd0427c82ae245d16d6e93d 2024-11-17T08:17:01,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742388_1564 (size=31413) 2024-11-17T08:17:01,011 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68e3358487792e1f6c40accb55544a0f/B of 68e3358487792e1f6c40accb55544a0f into 28a5fa720bd0427c82ae245d16d6e93d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:17:01,011 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68e3358487792e1f6c40accb55544a0f: 2024-11-17T08:17:01,011 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., storeName=68e3358487792e1f6c40accb55544a0f/B, priority=13, startTime=1731831420966; duration=0sec 2024-11-17T08:17:01,011 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:17:01,011 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68e3358487792e1f6c40accb55544a0f:B 2024-11-17T08:17:01,011 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:17:01,012 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:17:01,012 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): 68e3358487792e1f6c40accb55544a0f/C is initiating minor compaction (all files) 2024-11-17T08:17:01,012 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68e3358487792e1f6c40accb55544a0f/C in TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:01,012 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/e2aa165869614511aa94f1bc65d44673, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/2ce4cf4653be4096afa12a279a5c13ba, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/71516fe5b9db45ce8d97648ad1a3646b] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp, totalSize=35.5 K 2024-11-17T08:17:01,014 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting e2aa165869614511aa94f1bc65d44673, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731831417126 2024-11-17T08:17:01,016 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ce4cf4653be4096afa12a279a5c13ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731831417778 2024-11-17T08:17:01,016 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 71516fe5b9db45ce8d97648ad1a3646b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731831418957 2024-11-17T08:17:01,019 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/47e33686176f490eaf818df9900cabba as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/47e33686176f490eaf818df9900cabba 2024-11-17T08:17:01,027 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68e3358487792e1f6c40accb55544a0f#C#compaction#476 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:17:01,027 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/7bbb25b6f4aa4d57ba8d8baf3ac25276 is 50, key is test_row_0/C:col10/1731831420096/Put/seqid=0 2024-11-17T08:17:01,032 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68e3358487792e1f6c40accb55544a0f/A of 68e3358487792e1f6c40accb55544a0f into 47e33686176f490eaf818df9900cabba(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:17:01,032 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68e3358487792e1f6c40accb55544a0f: 2024-11-17T08:17:01,032 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., storeName=68e3358487792e1f6c40accb55544a0f/A, priority=13, startTime=1731831420966; duration=0sec 2024-11-17T08:17:01,032 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:17:01,032 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68e3358487792e1f6c40accb55544a0f:A 2024-11-17T08:17:01,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742389_1565 (size=12459) 2024-11-17T08:17:01,051 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/7bbb25b6f4aa4d57ba8d8baf3ac25276 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/7bbb25b6f4aa4d57ba8d8baf3ac25276 2024-11-17T08:17:01,062 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68e3358487792e1f6c40accb55544a0f/C of 68e3358487792e1f6c40accb55544a0f into 7bbb25b6f4aa4d57ba8d8baf3ac25276(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:17:01,062 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68e3358487792e1f6c40accb55544a0f: 2024-11-17T08:17:01,062 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., storeName=68e3358487792e1f6c40accb55544a0f/C, priority=13, startTime=1731831420966; duration=0sec 2024-11-17T08:17:01,062 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:17:01,062 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68e3358487792e1f6c40accb55544a0f:C 2024-11-17T08:17:01,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T08:17:01,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:01,094 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 68e3358487792e1f6c40accb55544a0f 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-17T08:17:01,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=A 2024-11-17T08:17:01,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:17:01,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=B 2024-11-17T08:17:01,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:17:01,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=C 2024-11-17T08:17:01,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:17:01,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117aaf8595c7ccd4b2da55658a67436845b_68e3358487792e1f6c40accb55544a0f is 50, key is test_row_0/A:col10/1731831420122/Put/seqid=0 2024-11-17T08:17:01,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742390_1566 (size=12304) 2024-11-17T08:17:01,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:01,126 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117aaf8595c7ccd4b2da55658a67436845b_68e3358487792e1f6c40accb55544a0f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117aaf8595c7ccd4b2da55658a67436845b_68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:01,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/218195681e9746f5912a1aed5c33f206, store: [table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:17:01,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/218195681e9746f5912a1aed5c33f206 is 175, key is test_row_0/A:col10/1731831420122/Put/seqid=0 2024-11-17T08:17:01,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742391_1567 (size=31105) 2024-11-17T08:17:01,143 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/218195681e9746f5912a1aed5c33f206 2024-11-17T08:17:01,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/023c48adb7fa443cbf6d07f646e1033d is 50, key is test_row_0/B:col10/1731831420122/Put/seqid=0 2024-11-17T08:17:01,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T08:17:01,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742392_1568 (size=12151) 2024-11-17T08:17:01,171 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/023c48adb7fa443cbf6d07f646e1033d 2024-11-17T08:17:01,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/62295450a60f4bc1a2271925d80faa51 is 50, key is test_row_0/C:col10/1731831420122/Put/seqid=0 2024-11-17T08:17:01,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742393_1569 (size=12151) 2024-11-17T08:17:01,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:01,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:17:01,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:01,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831481280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:01,284 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:01,284 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:01,284 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:01,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:01,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831481282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:01,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:01,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:01,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:01,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:01,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831481282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:01,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:01,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:01,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:01,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:01,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831481393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:01,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:01,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:01,397 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:01,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831481393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:01,397 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:01,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831481393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:01,398 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:01,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:01,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:01,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:01,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:01,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:01,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:01,585 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/62295450a60f4bc1a2271925d80faa51 2024-11-17T08:17:01,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/218195681e9746f5912a1aed5c33f206 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/218195681e9746f5912a1aed5c33f206 2024-11-17T08:17:01,591 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/218195681e9746f5912a1aed5c33f206, entries=150, sequenceid=156, filesize=30.4 K 2024-11-17T08:17:01,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/023c48adb7fa443cbf6d07f646e1033d as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/023c48adb7fa443cbf6d07f646e1033d 2024-11-17T08:17:01,594 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/023c48adb7fa443cbf6d07f646e1033d, entries=150, sequenceid=156, filesize=11.9 K 2024-11-17T08:17:01,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/62295450a60f4bc1a2271925d80faa51 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/62295450a60f4bc1a2271925d80faa51 2024-11-17T08:17:01,597 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/62295450a60f4bc1a2271925d80faa51, entries=150, sequenceid=156, filesize=11.9 K 2024-11-17T08:17:01,597 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 68e3358487792e1f6c40accb55544a0f in 503ms, sequenceid=156, compaction requested=false 2024-11-17T08:17:01,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 68e3358487792e1f6c40accb55544a0f: 2024-11-17T08:17:01,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:01,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-17T08:17:01,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-17T08:17:01,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-17T08:17:01,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5720 sec 2024-11-17T08:17:01,601 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.5760 sec 2024-11-17T08:17:01,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:01,607 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 68e3358487792e1f6c40accb55544a0f 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-17T08:17:01,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=A 2024-11-17T08:17:01,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:17:01,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=B 2024-11-17T08:17:01,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:17:01,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=C 2024-11-17T08:17:01,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:17:01,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c704901978174e6ab5f0754180680713_68e3358487792e1f6c40accb55544a0f is 50, key is test_row_0/A:col10/1731831421607/Put/seqid=0 2024-11-17T08:17:01,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742394_1570 (size=12304) 2024-11-17T08:17:01,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:01,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831481626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:01,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:01,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:01,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:01,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:01,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831481629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:01,632 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:01,632 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:01,632 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:01,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:01,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831481630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:01,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:01,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:01,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:01,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:01,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831481734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:01,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:01,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:01,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831481734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:01,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:01,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:01,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:01,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:01,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:01,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:01,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831481744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:01,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:01,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:01,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:01,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:01,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831481943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:01,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:01,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:01,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:01,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:01,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831481943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:01,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:01,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:01,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:01,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:01,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831481954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:01,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:01,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:01,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:02,015 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:02,018 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c704901978174e6ab5f0754180680713_68e3358487792e1f6c40accb55544a0f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c704901978174e6ab5f0754180680713_68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:02,019 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/fadb830277444cf39925890829b3eb71, store: [table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:17:02,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/fadb830277444cf39925890829b3eb71 is 175, key is test_row_0/A:col10/1731831421607/Put/seqid=0 2024-11-17T08:17:02,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742395_1571 (size=31105) 2024-11-17T08:17:02,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T08:17:02,174 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:17:02,174 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:17:02,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-17T08:17:02,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T08:17:02,176 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:17:02,176 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:17:02,177 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:17:02,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:02,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831482254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:02,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:02,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:02,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:02,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:02,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831482254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:02,256 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:02,256 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:02,256 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:02,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:02,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831482264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:02,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:02,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:02,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:02,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T08:17:02,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T08:17:02,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:02,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:17:02,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:02,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:02,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:02,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:02,423 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=173, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/fadb830277444cf39925890829b3eb71 2024-11-17T08:17:02,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/6583cd51b28c49b4ae2c48676642d6f6 is 50, key is test_row_0/B:col10/1731831421607/Put/seqid=0 2024-11-17T08:17:02,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742396_1572 (size=12151) 2024-11-17T08:17:02,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T08:17:02,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:02,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:17:02,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:02,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:02,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:02,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:02,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T08:17:02,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T08:17:02,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:02,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:17:02,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:02,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:02,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:02,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:02,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:02,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831482764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:02,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:02,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:02,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:02,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:02,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831482764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:02,767 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:02,768 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:02,768 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:02,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:02,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831482773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:02,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:02,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:02,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:02,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T08:17:02,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:02,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:17:02,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:02,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:02,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:02,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:02,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T08:17:02,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/6583cd51b28c49b4ae2c48676642d6f6 2024-11-17T08:17:02,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/2b5e4d26737d4027a864c93440d05aa1 is 50, key is test_row_0/C:col10/1731831421607/Put/seqid=0 2024-11-17T08:17:02,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742397_1573 (size=12151) 2024-11-17T08:17:02,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T08:17:02,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:02,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:17:02,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:02,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:02,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:02,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:03,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T08:17:03,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:03,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:17:03,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:03,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:03,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:03,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:03,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T08:17:03,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:03,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:17:03,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:03,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:03,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:03,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:03,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/2b5e4d26737d4027a864c93440d05aa1 2024-11-17T08:17:03,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/fadb830277444cf39925890829b3eb71 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/fadb830277444cf39925890829b3eb71 2024-11-17T08:17:03,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/fadb830277444cf39925890829b3eb71, entries=150, sequenceid=173, filesize=30.4 K 2024-11-17T08:17:03,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/6583cd51b28c49b4ae2c48676642d6f6 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/6583cd51b28c49b4ae2c48676642d6f6 2024-11-17T08:17:03,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/6583cd51b28c49b4ae2c48676642d6f6, entries=150, sequenceid=173, filesize=11.9 K 2024-11-17T08:17:03,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/2b5e4d26737d4027a864c93440d05aa1 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/2b5e4d26737d4027a864c93440d05aa1 2024-11-17T08:17:03,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/2b5e4d26737d4027a864c93440d05aa1, entries=150, sequenceid=173, filesize=11.9 K 2024-11-17T08:17:03,278 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 68e3358487792e1f6c40accb55544a0f in 1671ms, sequenceid=173, compaction requested=true 2024-11-17T08:17:03,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 68e3358487792e1f6c40accb55544a0f: 2024-11-17T08:17:03,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68e3358487792e1f6c40accb55544a0f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:17:03,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:17:03,278 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:17:03,278 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:17:03,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68e3358487792e1f6c40accb55544a0f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:17:03,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:17:03,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68e3358487792e1f6c40accb55544a0f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:17:03,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:17:03,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,279 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93623 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:17:03,279 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): 68e3358487792e1f6c40accb55544a0f/A is initiating minor compaction (all files) 2024-11-17T08:17:03,279 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68e3358487792e1f6c40accb55544a0f/A in TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:03,279 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/47e33686176f490eaf818df9900cabba, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/218195681e9746f5912a1aed5c33f206, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/fadb830277444cf39925890829b3eb71] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp, totalSize=91.4 K 2024-11-17T08:17:03,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,279 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:03,279 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. files: [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/47e33686176f490eaf818df9900cabba, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/218195681e9746f5912a1aed5c33f206, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/fadb830277444cf39925890829b3eb71] 2024-11-17T08:17:03,280 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:17:03,280 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): 68e3358487792e1f6c40accb55544a0f/B is initiating minor compaction (all files) 2024-11-17T08:17:03,280 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68e3358487792e1f6c40accb55544a0f/B in TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:03,280 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/28a5fa720bd0427c82ae245d16d6e93d, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/023c48adb7fa443cbf6d07f646e1033d, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/6583cd51b28c49b4ae2c48676642d6f6] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp, totalSize=35.9 K 2024-11-17T08:17:03,280 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47e33686176f490eaf818df9900cabba, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731831418957 2024-11-17T08:17:03,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,280 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 218195681e9746f5912a1aed5c33f206, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731831420111 2024-11-17T08:17:03,280 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 28a5fa720bd0427c82ae245d16d6e93d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731831418957 2024-11-17T08:17:03,281 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting fadb830277444cf39925890829b3eb71, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731831421281 2024-11-17T08:17:03,281 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 023c48adb7fa443cbf6d07f646e1033d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731831420111 2024-11-17T08:17:03,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,281 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 6583cd51b28c49b4ae2c48676642d6f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731831421281 2024-11-17T08:17:03,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,286 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:17:03,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,287 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68e3358487792e1f6c40accb55544a0f#B#compaction#483 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:17:03,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,288 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/e5550bb70dc5429cbcb0ab1332dad7ec is 50, key is test_row_0/B:col10/1731831421607/Put/seqid=0 2024-11-17T08:17:03,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,293 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111726367ffa4c754161a0e630ebc0e34b46_68e3358487792e1f6c40accb55544a0f store=[table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:17:03,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,295 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111726367ffa4c754161a0e630ebc0e34b46_68e3358487792e1f6c40accb55544a0f, store=[table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:17:03,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,295 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111726367ffa4c754161a0e630ebc0e34b46_68e3358487792e1f6c40accb55544a0f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:17:03,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742398_1574 (size=12561) 2024-11-17T08:17:03,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T08:17:03,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,325 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/e5550bb70dc5429cbcb0ab1332dad7ec as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/e5550bb70dc5429cbcb0ab1332dad7ec 2024-11-17T08:17:03,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742399_1575 (size=4469) 2024-11-17T08:17:03,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,330 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68e3358487792e1f6c40accb55544a0f/B of 68e3358487792e1f6c40accb55544a0f into e5550bb70dc5429cbcb0ab1332dad7ec(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:17:03,330 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68e3358487792e1f6c40accb55544a0f: 2024-11-17T08:17:03,330 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., storeName=68e3358487792e1f6c40accb55544a0f/B, priority=13, startTime=1731831423278; duration=0sec 2024-11-17T08:17:03,330 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:17:03,330 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68e3358487792e1f6c40accb55544a0f:B 2024-11-17T08:17:03,330 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:17:03,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,331 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:17:03,331 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): 68e3358487792e1f6c40accb55544a0f/C is initiating minor compaction (all files) 2024-11-17T08:17:03,331 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68e3358487792e1f6c40accb55544a0f/C in TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:03,331 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/7bbb25b6f4aa4d57ba8d8baf3ac25276, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/62295450a60f4bc1a2271925d80faa51, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/2b5e4d26737d4027a864c93440d05aa1] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp, totalSize=35.9 K 2024-11-17T08:17:03,332 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bbb25b6f4aa4d57ba8d8baf3ac25276, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731831418957 2024-11-17T08:17:03,332 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 62295450a60f4bc1a2271925d80faa51, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731831420111 2024-11-17T08:17:03,332 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b5e4d26737d4027a864c93440d05aa1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731831421281 2024-11-17T08:17:03,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,342 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68e3358487792e1f6c40accb55544a0f#C#compaction#485 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:17:03,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,343 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/f24e789e6c724fcc9f78ca4ad3f67fb8 is 50, key is test_row_0/C:col10/1731831421607/Put/seqid=0 2024-11-17T08:17:03,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742400_1576 (size=12561) 2024-11-17T08:17:03,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,364 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/f24e789e6c724fcc9f78ca4ad3f67fb8 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/f24e789e6c724fcc9f78ca4ad3f67fb8 2024-11-17T08:17:03,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,369 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68e3358487792e1f6c40accb55544a0f/C of 68e3358487792e1f6c40accb55544a0f into f24e789e6c724fcc9f78ca4ad3f67fb8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:17:03,369 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68e3358487792e1f6c40accb55544a0f: 2024-11-17T08:17:03,369 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., storeName=68e3358487792e1f6c40accb55544a0f/C, priority=13, startTime=1731831423278; duration=0sec 2024-11-17T08:17:03,369 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:17:03,369 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68e3358487792e1f6c40accb55544a0f:C 2024-11-17T08:17:03,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T08:17:03,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:03,400 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 68e3358487792e1f6c40accb55544a0f 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-17T08:17:03,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=A 2024-11-17T08:17:03,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:17:03,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=B 2024-11-17T08:17:03,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:17:03,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=C 2024-11-17T08:17:03,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:17:03,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111766b36180abe64ed4896ed6b1a5f24a82_68e3358487792e1f6c40accb55544a0f is 50, key is test_row_0/A:col10/1731831421629/Put/seqid=0 2024-11-17T08:17:03,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742401_1577 (size=9814) 2024-11-17T08:17:03,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,729 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68e3358487792e1f6c40accb55544a0f#A#compaction#484 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:17:03,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,729 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/a94655efbc334118bb45063ff0e9c25f is 175, key is test_row_0/A:col10/1731831421607/Put/seqid=0 2024-11-17T08:17:03,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742402_1578 (size=31515) 2024-11-17T08:17:03,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,747 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/a94655efbc334118bb45063ff0e9c25f as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/a94655efbc334118bb45063ff0e9c25f 2024-11-17T08:17:03,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,751 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68e3358487792e1f6c40accb55544a0f/A of 68e3358487792e1f6c40accb55544a0f into a94655efbc334118bb45063ff0e9c25f(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:17:03,751 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68e3358487792e1f6c40accb55544a0f: 2024-11-17T08:17:03,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,751 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., storeName=68e3358487792e1f6c40accb55544a0f/A, priority=13, startTime=1731831423278; duration=0sec 2024-11-17T08:17:03,751 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:17:03,751 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68e3358487792e1f6c40accb55544a0f:A 2024-11-17T08:17:03,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:17:03,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:03,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:03,842 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111766b36180abe64ed4896ed6b1a5f24a82_68e3358487792e1f6c40accb55544a0f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111766b36180abe64ed4896ed6b1a5f24a82_68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:03,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/1b767653ac284bad8127622f6525cf97, store: [table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:17:03,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/1b767653ac284bad8127622f6525cf97 is 175, key is test_row_0/A:col10/1731831421629/Put/seqid=0 2024-11-17T08:17:03,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742403_1579 (size=22461) 2024-11-17T08:17:03,847 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=191, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/1b767653ac284bad8127622f6525cf97 2024-11-17T08:17:03,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:03,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831483847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:03,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:03,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831483847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:03,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/696d10bd68e5452eb357030c7d6af9f0 is 50, key is test_row_0/B:col10/1731831421629/Put/seqid=0 2024-11-17T08:17:03,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:03,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831483849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:03,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:03,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:03,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:03,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:03,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:03,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:03,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:03,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:03,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:03,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742404_1580 (size=9757) 2024-11-17T08:17:03,871 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/696d10bd68e5452eb357030c7d6af9f0 2024-11-17T08:17:03,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/577cbb87e1aa48699bc2fd8e2e44f8ab is 50, key is test_row_0/C:col10/1731831421629/Put/seqid=0 2024-11-17T08:17:03,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742405_1581 (size=9757) 2024-11-17T08:17:03,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:03,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831483963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:03,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:03,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:03,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:03,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:03,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831483963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:03,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:03,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831483964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:03,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:03,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:03,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:03,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:03,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:03,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:04,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:04,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831484173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:04,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:04,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:04,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:04,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:04,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831484173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:04,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:04,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831484174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:04,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:04,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:04,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:04,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:04,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:04,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:04,282 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/577cbb87e1aa48699bc2fd8e2e44f8ab 2024-11-17T08:17:04,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/1b767653ac284bad8127622f6525cf97 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/1b767653ac284bad8127622f6525cf97 2024-11-17T08:17:04,287 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/1b767653ac284bad8127622f6525cf97, entries=100, sequenceid=191, filesize=21.9 K 2024-11-17T08:17:04,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/696d10bd68e5452eb357030c7d6af9f0 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/696d10bd68e5452eb357030c7d6af9f0 2024-11-17T08:17:04,291 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/696d10bd68e5452eb357030c7d6af9f0, entries=100, sequenceid=191, filesize=9.5 K 2024-11-17T08:17:04,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/577cbb87e1aa48699bc2fd8e2e44f8ab as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/577cbb87e1aa48699bc2fd8e2e44f8ab 2024-11-17T08:17:04,294 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/577cbb87e1aa48699bc2fd8e2e44f8ab, entries=100, sequenceid=191, filesize=9.5 K 2024-11-17T08:17:04,295 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 68e3358487792e1f6c40accb55544a0f in 895ms, sequenceid=191, compaction requested=false 2024-11-17T08:17:04,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 68e3358487792e1f6c40accb55544a0f: 2024-11-17T08:17:04,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:04,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-17T08:17:04,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-17T08:17:04,297 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-17T08:17:04,297 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1200 sec 2024-11-17T08:17:04,299 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 2.1240 sec 2024-11-17T08:17:04,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T08:17:04,324 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:17:04,324 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:17:04,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-17T08:17:04,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-17T08:17:04,326 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:17:04,327 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:17:04,327 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:17:04,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-17T08:17:04,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-17T08:17:04,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:04,479 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 68e3358487792e1f6c40accb55544a0f 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-17T08:17:04,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=A 2024-11-17T08:17:04,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:17:04,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=B 2024-11-17T08:17:04,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:17:04,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=C 2024-11-17T08:17:04,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:17:04,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:04,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:17:04,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170a73b43297eb459e97c79d4d749bf643_68e3358487792e1f6c40accb55544a0f is 50, key is test_row_0/A:col10/1731831423812/Put/seqid=0 2024-11-17T08:17:04,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742406_1582 (size=12304) 2024-11-17T08:17:04,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:04,496 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170a73b43297eb459e97c79d4d749bf643_68e3358487792e1f6c40accb55544a0f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170a73b43297eb459e97c79d4d749bf643_68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:04,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/18d8d2261f394111813c07774b67cf60, store: [table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:17:04,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/18d8d2261f394111813c07774b67cf60 is 175, key is test_row_0/A:col10/1731831423812/Put/seqid=0 2024-11-17T08:17:04,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742407_1583 (size=31105) 2024-11-17T08:17:04,502 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/18d8d2261f394111813c07774b67cf60 2024-11-17T08:17:04,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:04,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/340a34ced4be46039addf6bccb62d2e1 is 50, key is test_row_0/B:col10/1731831423812/Put/seqid=0 2024-11-17T08:17:04,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831484502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:04,508 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:04,508 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:04,508 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:04,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742408_1584 (size=12151) 2024-11-17T08:17:04,510 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/340a34ced4be46039addf6bccb62d2e1 2024-11-17T08:17:04,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:04,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831484506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:04,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:04,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831484507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:04,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:04,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:04,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:04,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:04,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:04,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:04,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/e239b0f3abae43a5b76d018cea5904df is 50, key is test_row_0/C:col10/1731831423812/Put/seqid=0 2024-11-17T08:17:04,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742409_1585 (size=12151) 2024-11-17T08:17:04,519 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/e239b0f3abae43a5b76d018cea5904df 2024-11-17T08:17:04,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/18d8d2261f394111813c07774b67cf60 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/18d8d2261f394111813c07774b67cf60 2024-11-17T08:17:04,534 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/18d8d2261f394111813c07774b67cf60, entries=150, sequenceid=208, filesize=30.4 K 2024-11-17T08:17:04,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/340a34ced4be46039addf6bccb62d2e1 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/340a34ced4be46039addf6bccb62d2e1 2024-11-17T08:17:04,537 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/340a34ced4be46039addf6bccb62d2e1, entries=150, sequenceid=208, filesize=11.9 K 2024-11-17T08:17:04,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/e239b0f3abae43a5b76d018cea5904df as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/e239b0f3abae43a5b76d018cea5904df 2024-11-17T08:17:04,540 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/e239b0f3abae43a5b76d018cea5904df, entries=150, sequenceid=208, filesize=11.9 K 2024-11-17T08:17:04,540 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 68e3358487792e1f6c40accb55544a0f in 61ms, sequenceid=208, compaction requested=true 2024-11-17T08:17:04,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 68e3358487792e1f6c40accb55544a0f: 2024-11-17T08:17:04,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:04,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-17T08:17:04,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-17T08:17:04,543 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-17T08:17:04,543 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 215 msec 2024-11-17T08:17:04,545 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 220 msec 2024-11-17T08:17:04,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:04,617 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 68e3358487792e1f6c40accb55544a0f 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-17T08:17:04,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=A 2024-11-17T08:17:04,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:17:04,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=B 2024-11-17T08:17:04,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:17:04,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=C 2024-11-17T08:17:04,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:17:04,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117dbe01fa0f2fe4b8db1d7613bc24febe7_68e3358487792e1f6c40accb55544a0f is 50, key is test_row_0/A:col10/1731831424498/Put/seqid=0 2024-11-17T08:17:04,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742410_1586 (size=14794) 2024-11-17T08:17:04,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:04,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831484636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:04,642 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:04,642 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:04,642 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:04,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-17T08:17:04,643 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:17:04,644 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:17:04,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-17T08:17:04,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-17T08:17:04,645 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:17:04,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:04,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831484641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:04,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:04,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:04,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:04,645 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:17:04,645 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:17:04,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:04,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831484642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:04,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:04,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:04,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:04,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:04,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831484744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:04,748 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:04,748 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:04,748 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:04,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-17T08:17:04,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:04,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831484753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:04,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:04,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:04,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831484753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:04,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:04,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:04,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:04,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:04,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:04,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-17T08:17:04,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:04,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:17:04,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:04,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:04,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:04,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:04,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-17T08:17:04,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:04,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:17:04,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:04,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:04,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:04,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:04,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:04,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831484954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:04,956 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:04,956 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:04,956 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:04,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-17T08:17:04,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:04,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831484963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:04,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:04,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:04,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:04,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831484963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:04,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:04,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:04,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:04,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:05,027 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:05,030 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117dbe01fa0f2fe4b8db1d7613bc24febe7_68e3358487792e1f6c40accb55544a0f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117dbe01fa0f2fe4b8db1d7613bc24febe7_68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:05,032 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/4b9d436c9e9249ef9139aea995e0362b, store: [table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:17:05,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/4b9d436c9e9249ef9139aea995e0362b is 175, key is test_row_0/A:col10/1731831424498/Put/seqid=0 2024-11-17T08:17:05,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742411_1587 (size=39749) 2024-11-17T08:17:05,039 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=226, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/4b9d436c9e9249ef9139aea995e0362b 2024-11-17T08:17:05,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/87ca1d874fa24753a8116e440eedbc19 is 50, key is test_row_0/B:col10/1731831424498/Put/seqid=0 2024-11-17T08:17:05,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742412_1588 (size=12151) 2024-11-17T08:17:05,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/87ca1d874fa24753a8116e440eedbc19 2024-11-17T08:17:05,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/9c631045c91340ac9634ccc134710c4d is 50, key is test_row_0/C:col10/1731831424498/Put/seqid=0 2024-11-17T08:17:05,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742413_1589 (size=12151) 2024-11-17T08:17:05,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-17T08:17:05,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:05,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:17:05,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:05,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:05,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:05,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:05,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-17T08:17:05,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:05,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:17:05,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:05,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:05,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:05,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:05,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:05,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831485264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:05,267 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:05,267 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:05,267 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:05,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-17T08:17:05,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:05,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831485274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:05,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:05,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:05,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:05,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:05,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831485274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:05,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:05,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:05,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:05,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-17T08:17:05,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:05,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:17:05,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:05,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:05,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:05,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:05,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/9c631045c91340ac9634ccc134710c4d 2024-11-17T08:17:05,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/4b9d436c9e9249ef9139aea995e0362b as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/4b9d436c9e9249ef9139aea995e0362b 2024-11-17T08:17:05,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/4b9d436c9e9249ef9139aea995e0362b, entries=200, sequenceid=226, filesize=38.8 K 2024-11-17T08:17:05,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/87ca1d874fa24753a8116e440eedbc19 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/87ca1d874fa24753a8116e440eedbc19 2024-11-17T08:17:05,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/87ca1d874fa24753a8116e440eedbc19, entries=150, sequenceid=226, filesize=11.9 K 2024-11-17T08:17:05,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/9c631045c91340ac9634ccc134710c4d as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/9c631045c91340ac9634ccc134710c4d 2024-11-17T08:17:05,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/9c631045c91340ac9634ccc134710c4d, entries=150, sequenceid=226, filesize=11.9 K 2024-11-17T08:17:05,485 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 68e3358487792e1f6c40accb55544a0f in 867ms, sequenceid=226, compaction requested=true 2024-11-17T08:17:05,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 68e3358487792e1f6c40accb55544a0f: 2024-11-17T08:17:05,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68e3358487792e1f6c40accb55544a0f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:17:05,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:17:05,485 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T08:17:05,485 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T08:17:05,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68e3358487792e1f6c40accb55544a0f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:17:05,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:17:05,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68e3358487792e1f6c40accb55544a0f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:17:05,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:17:05,486 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46620 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T08:17:05,486 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 124830 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T08:17:05,486 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): 68e3358487792e1f6c40accb55544a0f/A is initiating minor compaction (all files) 2024-11-17T08:17:05,486 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): 68e3358487792e1f6c40accb55544a0f/B is initiating minor compaction (all files) 2024-11-17T08:17:05,486 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68e3358487792e1f6c40accb55544a0f/B in TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:05,486 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68e3358487792e1f6c40accb55544a0f/A in TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:05,486 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/a94655efbc334118bb45063ff0e9c25f, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/1b767653ac284bad8127622f6525cf97, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/18d8d2261f394111813c07774b67cf60, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/4b9d436c9e9249ef9139aea995e0362b] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp, totalSize=121.9 K 2024-11-17T08:17:05,486 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/e5550bb70dc5429cbcb0ab1332dad7ec, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/696d10bd68e5452eb357030c7d6af9f0, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/340a34ced4be46039addf6bccb62d2e1, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/87ca1d874fa24753a8116e440eedbc19] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp, totalSize=45.5 K 2024-11-17T08:17:05,486 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:05,486 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. files: [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/a94655efbc334118bb45063ff0e9c25f, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/1b767653ac284bad8127622f6525cf97, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/18d8d2261f394111813c07774b67cf60, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/4b9d436c9e9249ef9139aea995e0362b] 2024-11-17T08:17:05,486 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting a94655efbc334118bb45063ff0e9c25f, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731831421281 2024-11-17T08:17:05,486 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting e5550bb70dc5429cbcb0ab1332dad7ec, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731831421281 2024-11-17T08:17:05,486 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 696d10bd68e5452eb357030c7d6af9f0, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731831421624 2024-11-17T08:17:05,486 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b767653ac284bad8127622f6525cf97, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731831421624 2024-11-17T08:17:05,487 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 340a34ced4be46039addf6bccb62d2e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731831423812 2024-11-17T08:17:05,487 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18d8d2261f394111813c07774b67cf60, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731831423812 2024-11-17T08:17:05,487 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 87ca1d874fa24753a8116e440eedbc19, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731831424498 2024-11-17T08:17:05,487 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b9d436c9e9249ef9139aea995e0362b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731831424498 2024-11-17T08:17:05,491 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:17:05,495 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68e3358487792e1f6c40accb55544a0f#B#compaction#496 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:17:05,495 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/ad721ba78892444ca47f1d0006c4ca62 is 50, key is test_row_0/B:col10/1731831424498/Put/seqid=0 2024-11-17T08:17:05,506 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111715e0eee7602942b69a5e753f3df21fa9_68e3358487792e1f6c40accb55544a0f store=[table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:17:05,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742414_1590 (size=12697) 2024-11-17T08:17:05,509 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111715e0eee7602942b69a5e753f3df21fa9_68e3358487792e1f6c40accb55544a0f, store=[table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:17:05,509 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111715e0eee7602942b69a5e753f3df21fa9_68e3358487792e1f6c40accb55544a0f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:17:05,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742415_1591 (size=4469) 2024-11-17T08:17:05,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-17T08:17:05,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:05,560 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 68e3358487792e1f6c40accb55544a0f 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-17T08:17:05,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=A 2024-11-17T08:17:05,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:17:05,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=B 2024-11-17T08:17:05,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:17:05,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=C 2024-11-17T08:17:05,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:17:05,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d42a628275f644159d613858e2d331a7_68e3358487792e1f6c40accb55544a0f is 50, key is test_row_0/A:col10/1731831424635/Put/seqid=0 2024-11-17T08:17:05,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742416_1592 (size=12304) 2024-11-17T08:17:05,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:17:05,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:05,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-17T08:17:05,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:05,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831485832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:05,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:05,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831485832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:05,836 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:05,836 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:05,836 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:05,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:05,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:05,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:05,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:05,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831485832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:05,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:05,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:05,837 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:05,911 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/ad721ba78892444ca47f1d0006c4ca62 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/ad721ba78892444ca47f1d0006c4ca62 2024-11-17T08:17:05,912 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68e3358487792e1f6c40accb55544a0f#A#compaction#495 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:17:05,913 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/35c9211b0f024eb7bb87e76735b3a42f is 175, key is test_row_0/A:col10/1731831424498/Put/seqid=0 2024-11-17T08:17:05,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742417_1593 (size=31651) 2024-11-17T08:17:05,924 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 68e3358487792e1f6c40accb55544a0f/B of 68e3358487792e1f6c40accb55544a0f into ad721ba78892444ca47f1d0006c4ca62(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:17:05,924 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68e3358487792e1f6c40accb55544a0f: 2024-11-17T08:17:05,924 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., storeName=68e3358487792e1f6c40accb55544a0f/B, priority=12, startTime=1731831425485; duration=0sec 2024-11-17T08:17:05,924 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:17:05,924 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68e3358487792e1f6c40accb55544a0f:B 2024-11-17T08:17:05,924 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T08:17:05,926 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46620 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T08:17:05,926 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): 68e3358487792e1f6c40accb55544a0f/C is initiating minor compaction (all files) 2024-11-17T08:17:05,926 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68e3358487792e1f6c40accb55544a0f/C in TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:05,926 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/f24e789e6c724fcc9f78ca4ad3f67fb8, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/577cbb87e1aa48699bc2fd8e2e44f8ab, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/e239b0f3abae43a5b76d018cea5904df, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/9c631045c91340ac9634ccc134710c4d] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp, totalSize=45.5 K 2024-11-17T08:17:05,927 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting f24e789e6c724fcc9f78ca4ad3f67fb8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731831421281 2024-11-17T08:17:05,927 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 577cbb87e1aa48699bc2fd8e2e44f8ab, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731831421624 2024-11-17T08:17:05,927 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting e239b0f3abae43a5b76d018cea5904df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731831423812 2024-11-17T08:17:05,928 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c631045c91340ac9634ccc134710c4d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731831424498 2024-11-17T08:17:05,935 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68e3358487792e1f6c40accb55544a0f#C#compaction#498 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:17:05,935 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/0fcdc414d7c2438e9fcc8d48649a389a is 50, key is test_row_0/C:col10/1731831424498/Put/seqid=0 2024-11-17T08:17:05,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742418_1594 (size=12697) 2024-11-17T08:17:05,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831485943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:05,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831485943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:05,947 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:05,947 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:05,947 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:05,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:05,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:05,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:05,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831485943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:05,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:05,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:05,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:05,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:05,988 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d42a628275f644159d613858e2d331a7_68e3358487792e1f6c40accb55544a0f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d42a628275f644159d613858e2d331a7_68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:05,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/36e6642b8c974281a19407bc57e5fe22, store: [table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:17:05,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/36e6642b8c974281a19407bc57e5fe22 is 175, key is test_row_0/A:col10/1731831424635/Put/seqid=0 2024-11-17T08:17:05,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742419_1595 (size=31105) 2024-11-17T08:17:05,993 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=240, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/36e6642b8c974281a19407bc57e5fe22 2024-11-17T08:17:06,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/64a99727cc2c4e278dcc84f7607d7ed8 is 50, key is test_row_0/B:col10/1731831424635/Put/seqid=0 2024-11-17T08:17:06,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742420_1596 (size=12151) 2024-11-17T08:17:06,112 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T08:17:06,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831486153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:06,155 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:06,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:06,155 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:06,155 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831486153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:06,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:06,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:06,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:06,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831486153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:06,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:06,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:06,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:06,325 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/35c9211b0f024eb7bb87e76735b3a42f as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/35c9211b0f024eb7bb87e76735b3a42f 2024-11-17T08:17:06,330 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 68e3358487792e1f6c40accb55544a0f/A of 68e3358487792e1f6c40accb55544a0f into 35c9211b0f024eb7bb87e76735b3a42f(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:17:06,330 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68e3358487792e1f6c40accb55544a0f: 2024-11-17T08:17:06,330 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., storeName=68e3358487792e1f6c40accb55544a0f/A, priority=12, startTime=1731831425485; duration=0sec 2024-11-17T08:17:06,330 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:17:06,330 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68e3358487792e1f6c40accb55544a0f:A 2024-11-17T08:17:06,344 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/0fcdc414d7c2438e9fcc8d48649a389a as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/0fcdc414d7c2438e9fcc8d48649a389a 2024-11-17T08:17:06,351 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 68e3358487792e1f6c40accb55544a0f/C of 68e3358487792e1f6c40accb55544a0f into 0fcdc414d7c2438e9fcc8d48649a389a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:17:06,351 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68e3358487792e1f6c40accb55544a0f: 2024-11-17T08:17:06,351 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., storeName=68e3358487792e1f6c40accb55544a0f/C, priority=12, startTime=1731831425485; duration=0sec 2024-11-17T08:17:06,351 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:17:06,351 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68e3358487792e1f6c40accb55544a0f:C 2024-11-17T08:17:06,404 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/64a99727cc2c4e278dcc84f7607d7ed8 2024-11-17T08:17:06,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/2076d537208a4f14ad90a94948559433 is 50, key is test_row_0/C:col10/1731831424635/Put/seqid=0 2024-11-17T08:17:06,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742421_1597 (size=12151) 2024-11-17T08:17:06,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831486464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:06,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:06,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831486464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:06,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:06,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:06,466 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:06,466 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:06,466 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:06,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831486464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:06,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:06,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:06,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:06,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-17T08:17:06,814 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/2076d537208a4f14ad90a94948559433 2024-11-17T08:17:06,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/36e6642b8c974281a19407bc57e5fe22 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/36e6642b8c974281a19407bc57e5fe22 2024-11-17T08:17:06,819 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/36e6642b8c974281a19407bc57e5fe22, entries=150, sequenceid=240, filesize=30.4 K 2024-11-17T08:17:06,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/64a99727cc2c4e278dcc84f7607d7ed8 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/64a99727cc2c4e278dcc84f7607d7ed8 2024-11-17T08:17:06,823 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/64a99727cc2c4e278dcc84f7607d7ed8, entries=150, sequenceid=240, filesize=11.9 K 2024-11-17T08:17:06,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/2076d537208a4f14ad90a94948559433 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/2076d537208a4f14ad90a94948559433 2024-11-17T08:17:06,826 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/2076d537208a4f14ad90a94948559433, entries=150, sequenceid=240, filesize=11.9 K 2024-11-17T08:17:06,827 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 68e3358487792e1f6c40accb55544a0f in 1268ms, sequenceid=240, compaction requested=false 2024-11-17T08:17:06,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 68e3358487792e1f6c40accb55544a0f: 2024-11-17T08:17:06,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:06,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-17T08:17:06,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-17T08:17:06,830 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-17T08:17:06,830 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1830 sec 2024-11-17T08:17:06,831 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 2.1860 sec 2024-11-17T08:17:06,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:06,976 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 68e3358487792e1f6c40accb55544a0f 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-17T08:17:06,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=A 2024-11-17T08:17:06,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:17:06,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=B 2024-11-17T08:17:06,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:17:06,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=C 2024-11-17T08:17:06,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:17:06,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ae118cb64f09414bbe48b44fca0de871_68e3358487792e1f6c40accb55544a0f is 50, key is test_row_0/A:col10/1731831425802/Put/seqid=0 2024-11-17T08:17:06,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742422_1598 (size=12404) 2024-11-17T08:17:07,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:07,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831486997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:07,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:07,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:07,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:07,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:07,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831487003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:07,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831487003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:07,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:07,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:07,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:07,009 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:07,009 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:07,009 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:07,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:07,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831487114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:07,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:07,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:07,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:07,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831487114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:07,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:07,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:07,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:07,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:07,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:07,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831487114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:07,116 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:07,116 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:07,116 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:07,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:07,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831487324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:07,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:07,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:07,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831487324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:07,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:07,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:07,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:07,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:07,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:07,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:07,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831487324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:07,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:07,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:07,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:07,386 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:07,388 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ae118cb64f09414bbe48b44fca0de871_68e3358487792e1f6c40accb55544a0f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ae118cb64f09414bbe48b44fca0de871_68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:07,389 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/7e587348234841a0b8a481b26956638f, store: [table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:17:07,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/7e587348234841a0b8a481b26956638f is 175, key is test_row_0/A:col10/1731831425802/Put/seqid=0 2024-11-17T08:17:07,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742423_1599 (size=31205) 2024-11-17T08:17:07,399 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=262, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/7e587348234841a0b8a481b26956638f 2024-11-17T08:17:07,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/529ffa4975d54a6e9c5b893b9f241376 is 50, key is test_row_0/B:col10/1731831425802/Put/seqid=0 2024-11-17T08:17:07,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742424_1600 (size=12251) 2024-11-17T08:17:07,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:07,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831487633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:07,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:07,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:07,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:07,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831487633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:07,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:07,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:07,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:07,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:07,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:07,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831487633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:07,636 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:07,636 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:07,636 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:07,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/529ffa4975d54a6e9c5b893b9f241376 2024-11-17T08:17:07,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/feb47e085ece413dbbb95dc41a1b5759 is 50, key is test_row_0/C:col10/1731831425802/Put/seqid=0 2024-11-17T08:17:07,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742425_1601 (size=12251) 2024-11-17T08:17:08,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:08,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831488144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:08,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:08,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:08,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831488144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:08,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:08,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:08,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:08,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:08,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:08,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:08,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831488144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:08,150 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:08,150 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:08,150 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:08,226 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/feb47e085ece413dbbb95dc41a1b5759 2024-11-17T08:17:08,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/7e587348234841a0b8a481b26956638f as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/7e587348234841a0b8a481b26956638f 2024-11-17T08:17:08,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/7e587348234841a0b8a481b26956638f, entries=150, sequenceid=262, filesize=30.5 K 2024-11-17T08:17:08,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/529ffa4975d54a6e9c5b893b9f241376 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/529ffa4975d54a6e9c5b893b9f241376 2024-11-17T08:17:08,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/529ffa4975d54a6e9c5b893b9f241376, entries=150, sequenceid=262, filesize=12.0 K 2024-11-17T08:17:08,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/feb47e085ece413dbbb95dc41a1b5759 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/feb47e085ece413dbbb95dc41a1b5759 2024-11-17T08:17:08,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/feb47e085ece413dbbb95dc41a1b5759, entries=150, sequenceid=262, filesize=12.0 K 2024-11-17T08:17:08,243 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 68e3358487792e1f6c40accb55544a0f in 1267ms, sequenceid=262, compaction requested=true 2024-11-17T08:17:08,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 68e3358487792e1f6c40accb55544a0f: 2024-11-17T08:17:08,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68e3358487792e1f6c40accb55544a0f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:17:08,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:17:08,243 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:17:08,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68e3358487792e1f6c40accb55544a0f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:17:08,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:17:08,244 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:17:08,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68e3358487792e1f6c40accb55544a0f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:17:08,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:17:08,244 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93961 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:17:08,244 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): 68e3358487792e1f6c40accb55544a0f/A is initiating minor compaction (all files) 2024-11-17T08:17:08,244 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68e3358487792e1f6c40accb55544a0f/A in TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:08,245 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/35c9211b0f024eb7bb87e76735b3a42f, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/36e6642b8c974281a19407bc57e5fe22, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/7e587348234841a0b8a481b26956638f] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp, totalSize=91.8 K 2024-11-17T08:17:08,245 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:08,245 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. files: [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/35c9211b0f024eb7bb87e76735b3a42f, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/36e6642b8c974281a19407bc57e5fe22, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/7e587348234841a0b8a481b26956638f] 2024-11-17T08:17:08,247 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:17:08,247 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): 68e3358487792e1f6c40accb55544a0f/B is initiating minor compaction (all files) 2024-11-17T08:17:08,247 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68e3358487792e1f6c40accb55544a0f/B in TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:08,247 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/ad721ba78892444ca47f1d0006c4ca62, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/64a99727cc2c4e278dcc84f7607d7ed8, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/529ffa4975d54a6e9c5b893b9f241376] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp, totalSize=36.2 K 2024-11-17T08:17:08,248 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35c9211b0f024eb7bb87e76735b3a42f, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731831424498 2024-11-17T08:17:08,248 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36e6642b8c974281a19407bc57e5fe22, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731831424635 2024-11-17T08:17:08,248 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting ad721ba78892444ca47f1d0006c4ca62, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731831424498 2024-11-17T08:17:08,248 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 64a99727cc2c4e278dcc84f7607d7ed8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731831424635 2024-11-17T08:17:08,248 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e587348234841a0b8a481b26956638f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731831425802 2024-11-17T08:17:08,249 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 529ffa4975d54a6e9c5b893b9f241376, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731831425802 2024-11-17T08:17:08,257 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:17:08,259 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68e3358487792e1f6c40accb55544a0f#B#compaction#505 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:17:08,260 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/dba03b5baf894b06b5bb45df19813b27 is 50, key is test_row_0/B:col10/1731831425802/Put/seqid=0 2024-11-17T08:17:08,265 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117e4117d54a5b14bc6842bb8e987c11d49_68e3358487792e1f6c40accb55544a0f store=[table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:17:08,267 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117e4117d54a5b14bc6842bb8e987c11d49_68e3358487792e1f6c40accb55544a0f, store=[table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:17:08,267 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e4117d54a5b14bc6842bb8e987c11d49_68e3358487792e1f6c40accb55544a0f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:17:08,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742426_1602 (size=12899) 2024-11-17T08:17:08,292 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/dba03b5baf894b06b5bb45df19813b27 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/dba03b5baf894b06b5bb45df19813b27 2024-11-17T08:17:08,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742427_1603 (size=4469) 2024-11-17T08:17:08,309 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68e3358487792e1f6c40accb55544a0f/B of 68e3358487792e1f6c40accb55544a0f into dba03b5baf894b06b5bb45df19813b27(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:17:08,310 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68e3358487792e1f6c40accb55544a0f: 2024-11-17T08:17:08,310 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., storeName=68e3358487792e1f6c40accb55544a0f/B, priority=13, startTime=1731831428243; duration=0sec 2024-11-17T08:17:08,310 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:17:08,310 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68e3358487792e1f6c40accb55544a0f:B 2024-11-17T08:17:08,310 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:17:08,311 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:17:08,311 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): 68e3358487792e1f6c40accb55544a0f/C is initiating minor compaction (all files) 2024-11-17T08:17:08,311 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68e3358487792e1f6c40accb55544a0f/C in TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:08,311 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/0fcdc414d7c2438e9fcc8d48649a389a, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/2076d537208a4f14ad90a94948559433, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/feb47e085ece413dbbb95dc41a1b5759] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp, totalSize=36.2 K 2024-11-17T08:17:08,312 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fcdc414d7c2438e9fcc8d48649a389a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731831424498 2024-11-17T08:17:08,312 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 2076d537208a4f14ad90a94948559433, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731831424635 2024-11-17T08:17:08,312 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting feb47e085ece413dbbb95dc41a1b5759, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731831425802 2024-11-17T08:17:08,327 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68e3358487792e1f6c40accb55544a0f#C#compaction#506 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:17:08,327 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/731e77ea308f470db4b58d585a789aad is 50, key is test_row_0/C:col10/1731831425802/Put/seqid=0 2024-11-17T08:17:08,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742428_1604 (size=12899) 2024-11-17T08:17:08,695 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68e3358487792e1f6c40accb55544a0f#A#compaction#504 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:17:08,695 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/1c9b5492985f488babc6a9fe7748389e is 175, key is test_row_0/A:col10/1731831425802/Put/seqid=0 2024-11-17T08:17:08,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742429_1605 (size=31853) 2024-11-17T08:17:08,701 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/1c9b5492985f488babc6a9fe7748389e as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/1c9b5492985f488babc6a9fe7748389e 2024-11-17T08:17:08,706 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68e3358487792e1f6c40accb55544a0f/A of 68e3358487792e1f6c40accb55544a0f into 1c9b5492985f488babc6a9fe7748389e(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:17:08,706 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68e3358487792e1f6c40accb55544a0f: 2024-11-17T08:17:08,706 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., storeName=68e3358487792e1f6c40accb55544a0f/A, priority=13, startTime=1731831428243; duration=0sec 2024-11-17T08:17:08,706 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:17:08,706 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68e3358487792e1f6c40accb55544a0f:A 2024-11-17T08:17:08,753 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/731e77ea308f470db4b58d585a789aad as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/731e77ea308f470db4b58d585a789aad 2024-11-17T08:17:08,756 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68e3358487792e1f6c40accb55544a0f/C of 68e3358487792e1f6c40accb55544a0f into 731e77ea308f470db4b58d585a789aad(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:17:08,756 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68e3358487792e1f6c40accb55544a0f: 2024-11-17T08:17:08,756 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., storeName=68e3358487792e1f6c40accb55544a0f/C, priority=13, startTime=1731831428243; duration=0sec 2024-11-17T08:17:08,756 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:17:08,756 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68e3358487792e1f6c40accb55544a0f:C 2024-11-17T08:17:08,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-17T08:17:08,803 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:17:08,804 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:17:08,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-17T08:17:08,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-17T08:17:08,805 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:17:08,806 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:17:08,806 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:17:08,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-17T08:17:08,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-17T08:17:08,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:08,958 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing 68e3358487792e1f6c40accb55544a0f 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-17T08:17:08,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=A 2024-11-17T08:17:08,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:17:08,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=B 2024-11-17T08:17:08,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:17:08,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=C 2024-11-17T08:17:08,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:17:08,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411179f0d86f0ccb14849a57d43cc2240df76_68e3358487792e1f6c40accb55544a0f is 50, key is test_row_0/A:col10/1731831427001/Put/seqid=0 2024-11-17T08:17:08,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742430_1606 (size=12454) 2024-11-17T08:17:09,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-17T08:17:09,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:09,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:17:09,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:09,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831489185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:09,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:09,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:09,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:09,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:09,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:09,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831489185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:09,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831489186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:09,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:09,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:09,191 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:09,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:09,191 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:09,191 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:09,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831489293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:09,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:09,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:09,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831489293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:09,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:09,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:09,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:09,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831489294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:09,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:09,297 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:09,297 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:09,297 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:09,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:09,370 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411179f0d86f0ccb14849a57d43cc2240df76_68e3358487792e1f6c40accb55544a0f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411179f0d86f0ccb14849a57d43cc2240df76_68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:09,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/4e4e78c79e454925a84882e5fd4e82c2, store: [table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:17:09,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/4e4e78c79e454925a84882e5fd4e82c2 is 175, key is test_row_0/A:col10/1731831427001/Put/seqid=0 2024-11-17T08:17:09,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742431_1607 (size=31255) 2024-11-17T08:17:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-17T08:17:09,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:09,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831489503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:09,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:09,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:09,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831489503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:09,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:09,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:09,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:09,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831489504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:09,507 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:09,507 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:09,507 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:09,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:09,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:09,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:09,774 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=279, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/4e4e78c79e454925a84882e5fd4e82c2 2024-11-17T08:17:09,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/8001cea21cfb435bb015adee304cb5ab is 50, key is test_row_0/B:col10/1731831427001/Put/seqid=0 2024-11-17T08:17:09,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742432_1608 (size=12301) 2024-11-17T08:17:09,783 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/8001cea21cfb435bb015adee304cb5ab 2024-11-17T08:17:09,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/ef20a4413d0c455b85e157be25e812da is 50, key is test_row_0/C:col10/1731831427001/Put/seqid=0 2024-11-17T08:17:09,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742433_1609 (size=12301) 2024-11-17T08:17:09,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:09,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831489814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:09,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:09,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:09,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:09,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:09,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831489814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:09,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:09,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:09,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:09,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:09,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831489814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:09,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:09,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:09,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:09,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:09,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47582 deadline: 1731831489874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:09,876 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to e3d0adb87c01,40631,1731831247805 for 'test_row_0' in 68e3358487792e1f6c40accb55544a0f of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T08:17:09,876 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:09,876 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:09,876 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:09,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:09,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47604 deadline: 1731831489903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:09,906 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to e3d0adb87c01,40631,1731831247805 for 'test_row_2' in 68e3358487792e1f6c40accb55544a0f of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18249 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T08:17:09,906 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:09,907 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:09,907 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:09,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-17T08:17:10,192 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/ef20a4413d0c455b85e157be25e812da 2024-11-17T08:17:10,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/4e4e78c79e454925a84882e5fd4e82c2 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/4e4e78c79e454925a84882e5fd4e82c2 2024-11-17T08:17:10,198 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/4e4e78c79e454925a84882e5fd4e82c2, entries=150, sequenceid=279, filesize=30.5 K 2024-11-17T08:17:10,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/8001cea21cfb435bb015adee304cb5ab as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/8001cea21cfb435bb015adee304cb5ab 2024-11-17T08:17:10,202 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/8001cea21cfb435bb015adee304cb5ab, entries=150, sequenceid=279, filesize=12.0 K 2024-11-17T08:17:10,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/ef20a4413d0c455b85e157be25e812da as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/ef20a4413d0c455b85e157be25e812da 2024-11-17T08:17:10,208 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/ef20a4413d0c455b85e157be25e812da, entries=150, sequenceid=279, filesize=12.0 K 2024-11-17T08:17:10,209 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 68e3358487792e1f6c40accb55544a0f in 1251ms, sequenceid=279, compaction requested=false 2024-11-17T08:17:10,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for 68e3358487792e1f6c40accb55544a0f: 2024-11-17T08:17:10,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:10,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-17T08:17:10,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-17T08:17:10,213 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-17T08:17:10,213 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4050 sec 2024-11-17T08:17:10,215 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 1.4100 sec 2024-11-17T08:17:10,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:10,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 68e3358487792e1f6c40accb55544a0f 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-17T08:17:10,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=A 2024-11-17T08:17:10,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:17:10,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=B 2024-11-17T08:17:10,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:17:10,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=C 2024-11-17T08:17:10,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:17:10,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411171d6dc8e1f8324ce7a9edbf1756f2926c_68e3358487792e1f6c40accb55544a0f is 50, key is test_row_0/A:col10/1731831429172/Put/seqid=0 2024-11-17T08:17:10,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742434_1610 (size=12454) 2024-11-17T08:17:10,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:10,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831490338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:10,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:10,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:10,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:10,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:10,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831490341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:10,344 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:10,345 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:10,345 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:10,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:10,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831490342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:10,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:10,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:10,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:10,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:10,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831490453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:10,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:10,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831490454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:10,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:10,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831490454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:10,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:10,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:10,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:10,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:10,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:10,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:10,456 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:10,456 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:10,457 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:10,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:10,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831490663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:10,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:10,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:10,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:10,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831490663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:10,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:10,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:10,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:10,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:10,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:10,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831490663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:10,667 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:10,667 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:10,667 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:10,737 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:10,739 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411171d6dc8e1f8324ce7a9edbf1756f2926c_68e3358487792e1f6c40accb55544a0f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411171d6dc8e1f8324ce7a9edbf1756f2926c_68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:10,740 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/37af01e5c59a4eae986d1b0663fd5ed9, store: [table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:17:10,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/37af01e5c59a4eae986d1b0663fd5ed9 is 175, key is test_row_0/A:col10/1731831429172/Put/seqid=0 2024-11-17T08:17:10,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742435_1611 (size=31255) 2024-11-17T08:17:10,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-17T08:17:10,954 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:17:10,954 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-17T08:17:10,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-11-17T08:17:10,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-17T08:17:10,956 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T08:17:10,956 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T08:17:10,956 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T08:17:10,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:10,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:10,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831490974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:10,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831490974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:10,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:10,978 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:10,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:10,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:10,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831490974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:10,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:10,978 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:10,978 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:10,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:10,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:10,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:11,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-17T08:17:11,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-17T08:17:11,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:11,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:17:11,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:11,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:11,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:11,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:11,143 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=298, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/37af01e5c59a4eae986d1b0663fd5ed9 2024-11-17T08:17:11,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/c3ba6bf5d4dd4085851101fcc751b2b5 is 50, key is test_row_0/B:col10/1731831429172/Put/seqid=0 2024-11-17T08:17:11,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742436_1612 (size=12301) 2024-11-17T08:17:11,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-17T08:17:11,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:11,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:17:11,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:11,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:11,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:11,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:11,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-17T08:17:11,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-17T08:17:11,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:11,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:17:11,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:11,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:11,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:11,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:11,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:11,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1731831491483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:11,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:11,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:11,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:11,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:11,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1731831491484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:11,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T08:17:11,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40631 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47568 deadline: 1731831491484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:11,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:11,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:11,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:11,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 , the old value is region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:11,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T08:17:11,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., hostname=e3d0adb87c01,40631,1731831247805, seqNum=5 because the exception is null or not the one we care about 2024-11-17T08:17:11,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/c3ba6bf5d4dd4085851101fcc751b2b5 2024-11-17T08:17:11,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/d27d6fe8129b48b5bc9c40290ae7c394 is 50, key is test_row_0/C:col10/1731831429172/Put/seqid=0 2024-11-17T08:17:11,562 INFO [Thread-2365 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2365. 2024-11-17T08:17:11,562 DEBUG [Thread-2365 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:17:11,562 DEBUG [Thread-2365 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:17:11,562 DEBUG [Thread-2365 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:17:11,562 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:17:11,563 INFO [Thread-2363 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2363. 2024-11-17T08:17:11,563 DEBUG [Thread-2363 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:17:11,563 DEBUG [Thread-2363 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:17:11,563 DEBUG [Thread-2363 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:17:11,563 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:17:11,564 INFO [Thread-2367 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2367. 2024-11-17T08:17:11,564 DEBUG [Thread-2367 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:17:11,564 DEBUG [Thread-2367 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:17:11,564 DEBUG [Thread-2367 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:17:11,564 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:17:11,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742437_1613 (size=12301) 2024-11-17T08:17:11,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-17T08:17:11,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:11,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:17:11,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:11,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:11,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:11,567 INFO [Thread-2371 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2371. 2024-11-17T08:17:11,567 DEBUG [Thread-2371 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:17:11,567 DEBUG [Thread-2371 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:17:11,567 DEBUG [Thread-2371 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:17:11,567 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:17:11,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:11,567 INFO [Thread-2369 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2369. 2024-11-17T08:17:11,567 DEBUG [Thread-2369 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:17:11,567 DEBUG [Thread-2369 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:17:11,567 DEBUG [Thread-2369 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:17:11,567 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:17:11,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-17T08:17:11,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-17T08:17:11,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:11,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:17:11,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:11,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:11,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:11,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:11,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-17T08:17:11,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:11,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:17:11,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:11,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:11,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:11,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T08:17:11,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/d27d6fe8129b48b5bc9c40290ae7c394 2024-11-17T08:17:11,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/37af01e5c59a4eae986d1b0663fd5ed9 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/37af01e5c59a4eae986d1b0663fd5ed9 2024-11-17T08:17:11,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/37af01e5c59a4eae986d1b0663fd5ed9, entries=150, sequenceid=298, filesize=30.5 K 2024-11-17T08:17:11,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/c3ba6bf5d4dd4085851101fcc751b2b5 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/c3ba6bf5d4dd4085851101fcc751b2b5 2024-11-17T08:17:11,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/c3ba6bf5d4dd4085851101fcc751b2b5, entries=150, sequenceid=298, filesize=12.0 K 2024-11-17T08:17:11,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/d27d6fe8129b48b5bc9c40290ae7c394 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/d27d6fe8129b48b5bc9c40290ae7c394 2024-11-17T08:17:11,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/d27d6fe8129b48b5bc9c40290ae7c394, entries=150, sequenceid=298, filesize=12.0 K 2024-11-17T08:17:11,977 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 68e3358487792e1f6c40accb55544a0f in 1650ms, sequenceid=298, compaction requested=true 2024-11-17T08:17:11,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 68e3358487792e1f6c40accb55544a0f: 2024-11-17T08:17:11,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68e3358487792e1f6c40accb55544a0f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T08:17:11,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:17:11,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68e3358487792e1f6c40accb55544a0f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T08:17:11,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:17:11,978 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:17:11,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68e3358487792e1f6c40accb55544a0f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T08:17:11,978 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:17:11,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:17:11,978 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94363 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:17:11,978 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:17:11,978 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): 68e3358487792e1f6c40accb55544a0f/B is initiating minor compaction (all files) 2024-11-17T08:17:11,978 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1541): 68e3358487792e1f6c40accb55544a0f/A is initiating minor compaction (all files) 2024-11-17T08:17:11,978 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68e3358487792e1f6c40accb55544a0f/B in TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:11,978 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68e3358487792e1f6c40accb55544a0f/A in TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:11,978 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/1c9b5492985f488babc6a9fe7748389e, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/4e4e78c79e454925a84882e5fd4e82c2, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/37af01e5c59a4eae986d1b0663fd5ed9] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp, totalSize=92.2 K 2024-11-17T08:17:11,978 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/dba03b5baf894b06b5bb45df19813b27, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/8001cea21cfb435bb015adee304cb5ab, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/c3ba6bf5d4dd4085851101fcc751b2b5] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp, totalSize=36.6 K 2024-11-17T08:17:11,979 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:11,979 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. files: [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/1c9b5492985f488babc6a9fe7748389e, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/4e4e78c79e454925a84882e5fd4e82c2, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/37af01e5c59a4eae986d1b0663fd5ed9] 2024-11-17T08:17:11,979 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting dba03b5baf894b06b5bb45df19813b27, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731831425802 2024-11-17T08:17:11,979 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c9b5492985f488babc6a9fe7748389e, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731831425802 2024-11-17T08:17:11,979 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 8001cea21cfb435bb015adee304cb5ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731831426994 2024-11-17T08:17:11,979 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e4e78c79e454925a84882e5fd4e82c2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731831426994 2024-11-17T08:17:11,979 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting c3ba6bf5d4dd4085851101fcc751b2b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731831429172 2024-11-17T08:17:11,979 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37af01e5c59a4eae986d1b0663fd5ed9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731831429172 2024-11-17T08:17:11,983 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:17:11,985 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68e3358487792e1f6c40accb55544a0f#B#compaction#513 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:17:11,985 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117b59a146fe7ec40c2ab64ab51a9ca3991_68e3358487792e1f6c40accb55544a0f store=[table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:17:11,985 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/74ebeeb69e964ccba77a2c949093d658 is 50, key is test_row_0/B:col10/1731831429172/Put/seqid=0 2024-11-17T08:17:11,988 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117b59a146fe7ec40c2ab64ab51a9ca3991_68e3358487792e1f6c40accb55544a0f, store=[table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:17:11,988 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b59a146fe7ec40c2ab64ab51a9ca3991_68e3358487792e1f6c40accb55544a0f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:17:12,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742438_1614 (size=13051) 2024-11-17T08:17:12,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742439_1615 (size=4469) 2024-11-17T08:17:12,004 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68e3358487792e1f6c40accb55544a0f#A#compaction#514 average throughput is 1.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:17:12,004 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/8b74281bc21d404ba910900264e8181d is 175, key is test_row_0/A:col10/1731831429172/Put/seqid=0 2024-11-17T08:17:12,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742440_1616 (size=32005) 2024-11-17T08:17:12,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-17T08:17:12,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:12,024 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing 68e3358487792e1f6c40accb55544a0f 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-17T08:17:12,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=A 2024-11-17T08:17:12,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:17:12,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=B 2024-11-17T08:17:12,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:17:12,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=C 2024-11-17T08:17:12,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:17:12,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172800dc683152490c8d6d51b973bb91db_68e3358487792e1f6c40accb55544a0f is 50, key is test_row_0/A:col10/1731831430341/Put/seqid=0 2024-11-17T08:17:12,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742441_1617 (size=12454) 2024-11-17T08:17:12,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-17T08:17:12,407 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/74ebeeb69e964ccba77a2c949093d658 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/74ebeeb69e964ccba77a2c949093d658 2024-11-17T08:17:12,410 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/8b74281bc21d404ba910900264e8181d as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/8b74281bc21d404ba910900264e8181d 2024-11-17T08:17:12,410 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68e3358487792e1f6c40accb55544a0f/B of 68e3358487792e1f6c40accb55544a0f into 74ebeeb69e964ccba77a2c949093d658(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:17:12,410 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68e3358487792e1f6c40accb55544a0f: 2024-11-17T08:17:12,410 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., storeName=68e3358487792e1f6c40accb55544a0f/B, priority=13, startTime=1731831431978; duration=0sec 2024-11-17T08:17:12,410 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T08:17:12,410 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68e3358487792e1f6c40accb55544a0f:B 2024-11-17T08:17:12,410 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T08:17:12,411 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T08:17:12,411 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1541): 68e3358487792e1f6c40accb55544a0f/C is initiating minor compaction (all files) 2024-11-17T08:17:12,411 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68e3358487792e1f6c40accb55544a0f/C in TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:12,411 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/731e77ea308f470db4b58d585a789aad, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/ef20a4413d0c455b85e157be25e812da, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/d27d6fe8129b48b5bc9c40290ae7c394] into tmpdir=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp, totalSize=36.6 K 2024-11-17T08:17:12,411 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting 731e77ea308f470db4b58d585a789aad, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731831425802 2024-11-17T08:17:12,411 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting ef20a4413d0c455b85e157be25e812da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731831426994 2024-11-17T08:17:12,411 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] compactions.Compactor(225): Compacting d27d6fe8129b48b5bc9c40290ae7c394, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731831429172 2024-11-17T08:17:12,413 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68e3358487792e1f6c40accb55544a0f/A of 68e3358487792e1f6c40accb55544a0f into 8b74281bc21d404ba910900264e8181d(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:17:12,413 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68e3358487792e1f6c40accb55544a0f: 2024-11-17T08:17:12,413 INFO [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., storeName=68e3358487792e1f6c40accb55544a0f/A, priority=13, startTime=1731831431977; duration=0sec 2024-11-17T08:17:12,413 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:17:12,413 DEBUG [RS:0;e3d0adb87c01:40631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68e3358487792e1f6c40accb55544a0f:A 2024-11-17T08:17:12,416 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68e3358487792e1f6c40accb55544a0f#C#compaction#516 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T08:17:12,416 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/2c1fa9f5e68f44729ecaef7bfa87d2cd is 50, key is test_row_0/C:col10/1731831429172/Put/seqid=0 2024-11-17T08:17:12,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742442_1618 (size=13051) 2024-11-17T08:17:12,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:12,436 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411172800dc683152490c8d6d51b973bb91db_68e3358487792e1f6c40accb55544a0f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172800dc683152490c8d6d51b973bb91db_68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:12,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/e76c5e58567b42958d0897dc2f331b9a, store: [table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:17:12,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/e76c5e58567b42958d0897dc2f331b9a is 175, key is test_row_0/A:col10/1731831430341/Put/seqid=0 2024-11-17T08:17:12,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742443_1619 (size=31255) 2024-11-17T08:17:12,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40631 {}] regionserver.HRegion(8855): Flush requested on 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:12,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. as already flushing 2024-11-17T08:17:12,494 INFO [Thread-2360 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2360. 2024-11-17T08:17:12,495 DEBUG [Thread-2360 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:17:12,495 DEBUG [Thread-2360 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:17:12,495 DEBUG [Thread-2360 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:17:12,495 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:17:12,505 INFO [Thread-2358 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2358. 2024-11-17T08:17:12,505 INFO [Thread-2352 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2352. 2024-11-17T08:17:12,505 DEBUG [Thread-2358 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:17:12,505 DEBUG [Thread-2352 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:17:12,505 DEBUG [Thread-2358 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:17:12,505 DEBUG [Thread-2352 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:17:12,505 DEBUG [Thread-2352 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:17:12,505 DEBUG [Thread-2358 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:17:12,505 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:17:12,505 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:17:12,822 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/2c1fa9f5e68f44729ecaef7bfa87d2cd as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/2c1fa9f5e68f44729ecaef7bfa87d2cd 2024-11-17T08:17:12,825 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68e3358487792e1f6c40accb55544a0f/C of 68e3358487792e1f6c40accb55544a0f into 2c1fa9f5e68f44729ecaef7bfa87d2cd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T08:17:12,826 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68e3358487792e1f6c40accb55544a0f: 2024-11-17T08:17:12,826 INFO [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., storeName=68e3358487792e1f6c40accb55544a0f/C, priority=13, startTime=1731831431978; duration=0sec 2024-11-17T08:17:12,826 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T08:17:12,826 DEBUG [RS:0;e3d0adb87c01:40631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68e3358487792e1f6c40accb55544a0f:C 2024-11-17T08:17:12,840 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=311, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/e76c5e58567b42958d0897dc2f331b9a 2024-11-17T08:17:12,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/9b695465d3f445c5b1300a5ec8811664 is 50, key is test_row_0/B:col10/1731831430341/Put/seqid=0 2024-11-17T08:17:12,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742444_1620 (size=12301) 2024-11-17T08:17:12,848 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/9b695465d3f445c5b1300a5ec8811664 2024-11-17T08:17:12,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/e8090da770184351b00743fe0f23803c is 50, key is test_row_0/C:col10/1731831430341/Put/seqid=0 2024-11-17T08:17:12,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742445_1621 (size=12301) 2024-11-17T08:17:13,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-17T08:17:13,256 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/e8090da770184351b00743fe0f23803c 2024-11-17T08:17:13,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/e76c5e58567b42958d0897dc2f331b9a as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/e76c5e58567b42958d0897dc2f331b9a 2024-11-17T08:17:13,262 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/e76c5e58567b42958d0897dc2f331b9a, entries=150, sequenceid=311, filesize=30.5 K 2024-11-17T08:17:13,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/9b695465d3f445c5b1300a5ec8811664 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/9b695465d3f445c5b1300a5ec8811664 2024-11-17T08:17:13,265 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/9b695465d3f445c5b1300a5ec8811664, entries=150, sequenceid=311, filesize=12.0 K 2024-11-17T08:17:13,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/e8090da770184351b00743fe0f23803c as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/e8090da770184351b00743fe0f23803c 2024-11-17T08:17:13,268 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/e8090da770184351b00743fe0f23803c, entries=150, sequenceid=311, filesize=12.0 K 2024-11-17T08:17:13,269 INFO [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=20.13 KB/20610 for 68e3358487792e1f6c40accb55544a0f in 1245ms, sequenceid=311, compaction requested=false 2024-11-17T08:17:13,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for 68e3358487792e1f6c40accb55544a0f: 2024-11-17T08:17:13,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:13,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e3d0adb87c01:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-11-17T08:17:13,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-11-17T08:17:13,271 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-17T08:17:13,272 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3140 sec 2024-11-17T08:17:13,273 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 2.3180 sec 2024-11-17T08:17:15,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-17T08:17:15,124 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:17:19,934 INFO [Thread-2356 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2356. 2024-11-17T08:17:19,934 DEBUG [Thread-2356 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:17:19,934 DEBUG [Thread-2356 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:17:19,934 DEBUG [Thread-2356 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:17:19,934 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:17:19,955 INFO [Thread-2354 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2354. 2024-11-17T08:17:19,955 DEBUG [Thread-2354 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T08:17:19,955 DEBUG [Thread-2354 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:17:19,955 DEBUG [Thread-2354 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:17:19,955 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:17:19,955 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T08:17:19,955 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-17T08:17:19,955 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-17T08:17:19,955 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-11-17T08:17:19,955 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-17T08:17:19,955 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-17T08:17:19,955 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T08:17:19,955 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3152 2024-11-17T08:17:19,955 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3140 2024-11-17T08:17:19,955 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3106 2024-11-17T08:17:19,956 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3142 2024-11-17T08:17:19,956 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3156 2024-11-17T08:17:19,956 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T08:17:19,956 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T08:17:19,956 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T08:17:19,956 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:17:19,956 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:17:19,956 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:17:19,956 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-17T08:17:19,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T08:17:19,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-17T08:17:19,959 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731831439958"}]},"ts":"1731831439958"} 2024-11-17T08:17:19,959 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T08:17:19,959 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T08:17:19,960 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T08:17:19,961 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=68e3358487792e1f6c40accb55544a0f, UNASSIGN}] 2024-11-17T08:17:19,962 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=68e3358487792e1f6c40accb55544a0f, UNASSIGN 2024-11-17T08:17:19,962 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=174 updating hbase:meta row=68e3358487792e1f6c40accb55544a0f, regionState=CLOSING, regionLocation=e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:19,963 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=68e3358487792e1f6c40accb55544a0f, UNASSIGN because future has completed 2024-11-17T08:17:19,963 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T08:17:19,964 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE, hasLock=false; CloseRegionProcedure 68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805}] 2024-11-17T08:17:20,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-17T08:17:20,114 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(122): Close 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:20,114 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T08:17:20,115 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1722): Closing 68e3358487792e1f6c40accb55544a0f, disabling compactions & flushes 2024-11-17T08:17:20,115 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:20,115 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:20,115 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. after waiting 0 ms 2024-11-17T08:17:20,115 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:20,115 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(2902): Flushing 68e3358487792e1f6c40accb55544a0f 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-17T08:17:20,115 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=A 2024-11-17T08:17:20,115 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:17:20,115 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=B 2024-11-17T08:17:20,115 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:17:20,115 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68e3358487792e1f6c40accb55544a0f, store=C 2024-11-17T08:17:20,115 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T08:17:20,120 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111705f32d18c01c40e3bddd7b3d9e1fcd62_68e3358487792e1f6c40accb55544a0f is 50, key is test_row_0/A:col10/1731831439954/Put/seqid=0 2024-11-17T08:17:20,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742446_1622 (size=12454) 2024-11-17T08:17:20,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-17T08:17:20,523 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T08:17:20,526 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111705f32d18c01c40e3bddd7b3d9e1fcd62_68e3358487792e1f6c40accb55544a0f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111705f32d18c01c40e3bddd7b3d9e1fcd62_68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:20,527 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/1c78d1aaae044e8bb18620d7929302d4, store: [table=TestAcidGuarantees family=A region=68e3358487792e1f6c40accb55544a0f] 2024-11-17T08:17:20,527 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/1c78d1aaae044e8bb18620d7929302d4 is 175, key is test_row_0/A:col10/1731831439954/Put/seqid=0 2024-11-17T08:17:20,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742447_1623 (size=31255) 2024-11-17T08:17:20,530 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=322, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/1c78d1aaae044e8bb18620d7929302d4 2024-11-17T08:17:20,535 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/3d77cdd2766f47dc9cece0a8ed934cb8 is 50, key is test_row_0/B:col10/1731831439954/Put/seqid=0 2024-11-17T08:17:20,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742448_1624 (size=12301) 2024-11-17T08:17:20,542 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/3d77cdd2766f47dc9cece0a8ed934cb8 2024-11-17T08:17:20,549 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/3e367b193a99459f8f377b7cf554941e is 50, key is test_row_0/C:col10/1731831439954/Put/seqid=0 2024-11-17T08:17:20,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742449_1625 (size=12301) 2024-11-17T08:17:20,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-17T08:17:20,956 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/3e367b193a99459f8f377b7cf554941e 2024-11-17T08:17:20,960 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/A/1c78d1aaae044e8bb18620d7929302d4 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/1c78d1aaae044e8bb18620d7929302d4 2024-11-17T08:17:20,964 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/1c78d1aaae044e8bb18620d7929302d4, entries=150, sequenceid=322, filesize=30.5 K 2024-11-17T08:17:20,965 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/B/3d77cdd2766f47dc9cece0a8ed934cb8 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/3d77cdd2766f47dc9cece0a8ed934cb8 2024-11-17T08:17:20,968 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/3d77cdd2766f47dc9cece0a8ed934cb8, entries=150, sequenceid=322, filesize=12.0 K 2024-11-17T08:17:20,968 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/.tmp/C/3e367b193a99459f8f377b7cf554941e as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/3e367b193a99459f8f377b7cf554941e 2024-11-17T08:17:20,970 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/3e367b193a99459f8f377b7cf554941e, entries=150, sequenceid=322, filesize=12.0 K 2024-11-17T08:17:20,971 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 68e3358487792e1f6c40accb55544a0f in 856ms, sequenceid=322, compaction requested=true 2024-11-17T08:17:20,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/a86829658e9949f28fdb9ddfc090dba7, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/30fff306337d4bb58e5458dc754d5efd, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/b8785aeece1b4a4580ae7d2467ff77cf, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/defe01e0d1294dfa94ff7d22514ab2a3, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/1b5bafb627ca4344981204f6a90a7c73, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/eabf0b31fe6442a3bd10dede5fb3a855, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/d7df533ba0f94fb087b5a453efd69e30, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/21f7a7294c0c4a2d8a6a642b2db2b40f, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/5d3c8a3137434ee1b244441ecfc00875, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/47e33686176f490eaf818df9900cabba, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/218195681e9746f5912a1aed5c33f206, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/a94655efbc334118bb45063ff0e9c25f, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/fadb830277444cf39925890829b3eb71, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/1b767653ac284bad8127622f6525cf97, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/18d8d2261f394111813c07774b67cf60, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/4b9d436c9e9249ef9139aea995e0362b, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/35c9211b0f024eb7bb87e76735b3a42f, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/36e6642b8c974281a19407bc57e5fe22, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/1c9b5492985f488babc6a9fe7748389e, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/7e587348234841a0b8a481b26956638f, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/4e4e78c79e454925a84882e5fd4e82c2, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/37af01e5c59a4eae986d1b0663fd5ed9] to archive 2024-11-17T08:17:20,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T08:17:20,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/a86829658e9949f28fdb9ddfc090dba7 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/a86829658e9949f28fdb9ddfc090dba7 2024-11-17T08:17:20,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/30fff306337d4bb58e5458dc754d5efd to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/30fff306337d4bb58e5458dc754d5efd 2024-11-17T08:17:20,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/b8785aeece1b4a4580ae7d2467ff77cf to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/b8785aeece1b4a4580ae7d2467ff77cf 2024-11-17T08:17:20,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/defe01e0d1294dfa94ff7d22514ab2a3 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/defe01e0d1294dfa94ff7d22514ab2a3 2024-11-17T08:17:20,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/1b5bafb627ca4344981204f6a90a7c73 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/1b5bafb627ca4344981204f6a90a7c73 2024-11-17T08:17:20,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/eabf0b31fe6442a3bd10dede5fb3a855 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/eabf0b31fe6442a3bd10dede5fb3a855 2024-11-17T08:17:20,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/d7df533ba0f94fb087b5a453efd69e30 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/d7df533ba0f94fb087b5a453efd69e30 2024-11-17T08:17:20,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/21f7a7294c0c4a2d8a6a642b2db2b40f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/21f7a7294c0c4a2d8a6a642b2db2b40f 2024-11-17T08:17:20,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/5d3c8a3137434ee1b244441ecfc00875 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/5d3c8a3137434ee1b244441ecfc00875 2024-11-17T08:17:20,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/47e33686176f490eaf818df9900cabba to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/47e33686176f490eaf818df9900cabba 2024-11-17T08:17:20,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/218195681e9746f5912a1aed5c33f206 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/218195681e9746f5912a1aed5c33f206 2024-11-17T08:17:20,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/a94655efbc334118bb45063ff0e9c25f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/a94655efbc334118bb45063ff0e9c25f 2024-11-17T08:17:20,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/fadb830277444cf39925890829b3eb71 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/fadb830277444cf39925890829b3eb71 2024-11-17T08:17:20,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/1b767653ac284bad8127622f6525cf97 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/1b767653ac284bad8127622f6525cf97 2024-11-17T08:17:20,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/18d8d2261f394111813c07774b67cf60 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/18d8d2261f394111813c07774b67cf60 2024-11-17T08:17:20,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/4b9d436c9e9249ef9139aea995e0362b to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/4b9d436c9e9249ef9139aea995e0362b 2024-11-17T08:17:20,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/35c9211b0f024eb7bb87e76735b3a42f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/35c9211b0f024eb7bb87e76735b3a42f 2024-11-17T08:17:20,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/36e6642b8c974281a19407bc57e5fe22 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/36e6642b8c974281a19407bc57e5fe22 2024-11-17T08:17:20,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/1c9b5492985f488babc6a9fe7748389e to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/1c9b5492985f488babc6a9fe7748389e 2024-11-17T08:17:20,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/7e587348234841a0b8a481b26956638f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/7e587348234841a0b8a481b26956638f 2024-11-17T08:17:20,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/4e4e78c79e454925a84882e5fd4e82c2 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/4e4e78c79e454925a84882e5fd4e82c2 2024-11-17T08:17:20,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/37af01e5c59a4eae986d1b0663fd5ed9 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/37af01e5c59a4eae986d1b0663fd5ed9 2024-11-17T08:17:20,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/d7e369bc33304acf8c131c1a6f26d912, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/4cc7c38f8e9a4092995bdcc0964d216d, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/0f3de109dcea444dbaa0504eaa55fbc4, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/fdcecd8e2a3442a0b85bf2153bbe39b4, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/f7594801fa044550a0f0a3e98139836e, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/8ea14f811ce24a759d9016a4c4fcfd26, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/abd5284b9d93416aa669e7eb84054d2a, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/c8f626d32b4d4170bc6c06cae715140c, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/28a5fa720bd0427c82ae245d16d6e93d, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/31bd6a59006843f2b89035179444bfa9, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/023c48adb7fa443cbf6d07f646e1033d, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/e5550bb70dc5429cbcb0ab1332dad7ec, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/6583cd51b28c49b4ae2c48676642d6f6, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/696d10bd68e5452eb357030c7d6af9f0, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/340a34ced4be46039addf6bccb62d2e1, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/ad721ba78892444ca47f1d0006c4ca62, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/87ca1d874fa24753a8116e440eedbc19, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/64a99727cc2c4e278dcc84f7607d7ed8, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/dba03b5baf894b06b5bb45df19813b27, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/529ffa4975d54a6e9c5b893b9f241376, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/8001cea21cfb435bb015adee304cb5ab, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/c3ba6bf5d4dd4085851101fcc751b2b5] to archive 2024-11-17T08:17:20,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T08:17:20,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/d7e369bc33304acf8c131c1a6f26d912 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/d7e369bc33304acf8c131c1a6f26d912 2024-11-17T08:17:20,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/4cc7c38f8e9a4092995bdcc0964d216d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/4cc7c38f8e9a4092995bdcc0964d216d 2024-11-17T08:17:20,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/0f3de109dcea444dbaa0504eaa55fbc4 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/0f3de109dcea444dbaa0504eaa55fbc4 2024-11-17T08:17:20,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/fdcecd8e2a3442a0b85bf2153bbe39b4 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/fdcecd8e2a3442a0b85bf2153bbe39b4 2024-11-17T08:17:20,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/f7594801fa044550a0f0a3e98139836e to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/f7594801fa044550a0f0a3e98139836e 2024-11-17T08:17:21,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/8ea14f811ce24a759d9016a4c4fcfd26 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/8ea14f811ce24a759d9016a4c4fcfd26 2024-11-17T08:17:21,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/abd5284b9d93416aa669e7eb84054d2a to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/abd5284b9d93416aa669e7eb84054d2a 2024-11-17T08:17:21,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/c8f626d32b4d4170bc6c06cae715140c to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/c8f626d32b4d4170bc6c06cae715140c 2024-11-17T08:17:21,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/28a5fa720bd0427c82ae245d16d6e93d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/28a5fa720bd0427c82ae245d16d6e93d 2024-11-17T08:17:21,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/31bd6a59006843f2b89035179444bfa9 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/31bd6a59006843f2b89035179444bfa9 2024-11-17T08:17:21,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/023c48adb7fa443cbf6d07f646e1033d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/023c48adb7fa443cbf6d07f646e1033d 2024-11-17T08:17:21,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/e5550bb70dc5429cbcb0ab1332dad7ec to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/e5550bb70dc5429cbcb0ab1332dad7ec 2024-11-17T08:17:21,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/6583cd51b28c49b4ae2c48676642d6f6 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/6583cd51b28c49b4ae2c48676642d6f6 2024-11-17T08:17:21,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/696d10bd68e5452eb357030c7d6af9f0 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/696d10bd68e5452eb357030c7d6af9f0 2024-11-17T08:17:21,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/340a34ced4be46039addf6bccb62d2e1 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/340a34ced4be46039addf6bccb62d2e1 2024-11-17T08:17:21,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/ad721ba78892444ca47f1d0006c4ca62 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/ad721ba78892444ca47f1d0006c4ca62 2024-11-17T08:17:21,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/87ca1d874fa24753a8116e440eedbc19 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/87ca1d874fa24753a8116e440eedbc19 2024-11-17T08:17:21,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/64a99727cc2c4e278dcc84f7607d7ed8 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/64a99727cc2c4e278dcc84f7607d7ed8 2024-11-17T08:17:21,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/dba03b5baf894b06b5bb45df19813b27 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/dba03b5baf894b06b5bb45df19813b27 2024-11-17T08:17:21,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/529ffa4975d54a6e9c5b893b9f241376 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/529ffa4975d54a6e9c5b893b9f241376 2024-11-17T08:17:21,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/8001cea21cfb435bb015adee304cb5ab to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/8001cea21cfb435bb015adee304cb5ab 2024-11-17T08:17:21,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/c3ba6bf5d4dd4085851101fcc751b2b5 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/c3ba6bf5d4dd4085851101fcc751b2b5 2024-11-17T08:17:21,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/ba1537e2fb8e422e97d825e9681028b8, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/3e7d3c8c3e9a46fbbe2e25070d94cdc5, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/122ba7313e664c30bf9e7dea0ca40805, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/6e79efe437e84845bf618ff7e3c3a3d9, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/8ff196210b754475b4245134d5c17271, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/e2aa165869614511aa94f1bc65d44673, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/b28fe82ee72b40fb9cd15825ea7eb593, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/2ce4cf4653be4096afa12a279a5c13ba, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/7bbb25b6f4aa4d57ba8d8baf3ac25276, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/71516fe5b9db45ce8d97648ad1a3646b, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/62295450a60f4bc1a2271925d80faa51, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/f24e789e6c724fcc9f78ca4ad3f67fb8, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/2b5e4d26737d4027a864c93440d05aa1, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/577cbb87e1aa48699bc2fd8e2e44f8ab, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/e239b0f3abae43a5b76d018cea5904df, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/0fcdc414d7c2438e9fcc8d48649a389a, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/9c631045c91340ac9634ccc134710c4d, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/2076d537208a4f14ad90a94948559433, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/731e77ea308f470db4b58d585a789aad, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/feb47e085ece413dbbb95dc41a1b5759, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/ef20a4413d0c455b85e157be25e812da, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/d27d6fe8129b48b5bc9c40290ae7c394] to archive 2024-11-17T08:17:21,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T08:17:21,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/ba1537e2fb8e422e97d825e9681028b8 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/ba1537e2fb8e422e97d825e9681028b8 2024-11-17T08:17:21,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/3e7d3c8c3e9a46fbbe2e25070d94cdc5 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/3e7d3c8c3e9a46fbbe2e25070d94cdc5 2024-11-17T08:17:21,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/122ba7313e664c30bf9e7dea0ca40805 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/122ba7313e664c30bf9e7dea0ca40805 2024-11-17T08:17:21,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/6e79efe437e84845bf618ff7e3c3a3d9 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/6e79efe437e84845bf618ff7e3c3a3d9 2024-11-17T08:17:21,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/8ff196210b754475b4245134d5c17271 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/8ff196210b754475b4245134d5c17271 2024-11-17T08:17:21,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/e2aa165869614511aa94f1bc65d44673 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/e2aa165869614511aa94f1bc65d44673 2024-11-17T08:17:21,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/b28fe82ee72b40fb9cd15825ea7eb593 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/b28fe82ee72b40fb9cd15825ea7eb593 2024-11-17T08:17:21,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/2ce4cf4653be4096afa12a279a5c13ba to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/2ce4cf4653be4096afa12a279a5c13ba 2024-11-17T08:17:21,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/7bbb25b6f4aa4d57ba8d8baf3ac25276 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/7bbb25b6f4aa4d57ba8d8baf3ac25276 2024-11-17T08:17:21,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/71516fe5b9db45ce8d97648ad1a3646b to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/71516fe5b9db45ce8d97648ad1a3646b 2024-11-17T08:17:21,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/62295450a60f4bc1a2271925d80faa51 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/62295450a60f4bc1a2271925d80faa51 2024-11-17T08:17:21,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/f24e789e6c724fcc9f78ca4ad3f67fb8 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/f24e789e6c724fcc9f78ca4ad3f67fb8 2024-11-17T08:17:21,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/2b5e4d26737d4027a864c93440d05aa1 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/2b5e4d26737d4027a864c93440d05aa1 2024-11-17T08:17:21,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/577cbb87e1aa48699bc2fd8e2e44f8ab to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/577cbb87e1aa48699bc2fd8e2e44f8ab 2024-11-17T08:17:21,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/e239b0f3abae43a5b76d018cea5904df to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/e239b0f3abae43a5b76d018cea5904df 2024-11-17T08:17:21,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/0fcdc414d7c2438e9fcc8d48649a389a to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/0fcdc414d7c2438e9fcc8d48649a389a 2024-11-17T08:17:21,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/9c631045c91340ac9634ccc134710c4d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/9c631045c91340ac9634ccc134710c4d 2024-11-17T08:17:21,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/2076d537208a4f14ad90a94948559433 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/2076d537208a4f14ad90a94948559433 2024-11-17T08:17:21,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/731e77ea308f470db4b58d585a789aad to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/731e77ea308f470db4b58d585a789aad 2024-11-17T08:17:21,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/feb47e085ece413dbbb95dc41a1b5759 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/feb47e085ece413dbbb95dc41a1b5759 2024-11-17T08:17:21,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/ef20a4413d0c455b85e157be25e812da to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/ef20a4413d0c455b85e157be25e812da 2024-11-17T08:17:21,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/d27d6fe8129b48b5bc9c40290ae7c394 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/d27d6fe8129b48b5bc9c40290ae7c394 2024-11-17T08:17:21,040 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/recovered.edits/325.seqid, newMaxSeqId=325, maxSeqId=4 2024-11-17T08:17:21,040 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. 2024-11-17T08:17:21,041 DEBUG [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1676): Region close journal for 68e3358487792e1f6c40accb55544a0f: Waiting for close lock at 1731831440115Running coprocessor pre-close hooks at 1731831440115Disabling compacts and flushes for region at 1731831440115Disabling writes for close at 1731831440115Obtaining lock to block concurrent updates at 1731831440115Preparing flush snapshotting stores in 68e3358487792e1f6c40accb55544a0f at 1731831440115Finished memstore snapshotting TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731831440115Flushing stores of TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f. at 1731831440116 (+1 ms)Flushing 68e3358487792e1f6c40accb55544a0f/A: creating writer at 1731831440116Flushing mob file 68e3358487792e1f6c40accb55544a0f/A: appending metadata at 1731831440120 (+4 ms)Flushing mob file 68e3358487792e1f6c40accb55544a0f/A: closing flushed file at 1731831440120Flushing 68e3358487792e1f6c40accb55544a0f/A: appending metadata at 1731831440527 (+407 ms)Flushing 68e3358487792e1f6c40accb55544a0f/A: closing flushed file at 1731831440527Flushing 68e3358487792e1f6c40accb55544a0f/B: creating writer at 1731831440532 (+5 ms)Flushing 68e3358487792e1f6c40accb55544a0f/B: appending metadata at 1731831440535 (+3 ms)Flushing 68e3358487792e1f6c40accb55544a0f/B: closing flushed file at 1731831440535Flushing 68e3358487792e1f6c40accb55544a0f/C: creating writer at 1731831440545 (+10 ms)Flushing 68e3358487792e1f6c40accb55544a0f/C: appending metadata at 1731831440548 (+3 ms)Flushing 68e3358487792e1f6c40accb55544a0f/C: closing flushed file at 1731831440548Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3dfb57e: reopening flushed file at 1731831440959 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9899017: reopening flushed file at 1731831440964 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e1a6366: reopening flushed file at 1731831440968 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 68e3358487792e1f6c40accb55544a0f in 856ms, sequenceid=322, compaction requested=true at 1731831440971 (+3 ms)Writing region close event to WAL at 1731831441037 (+66 ms)Running coprocessor post-close hooks at 1731831441040 (+3 ms)Closed at 1731831441040 2024-11-17T08:17:21,042 INFO [RS_CLOSE_REGION-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(157): Closed 68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:21,043 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=174 updating hbase:meta row=68e3358487792e1f6c40accb55544a0f, regionState=CLOSED 2024-11-17T08:17:21,044 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=175, ppid=174, state=RUNNABLE, hasLock=false; CloseRegionProcedure 68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 because future has completed 2024-11-17T08:17:21,045 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-11-17T08:17:21,046 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; CloseRegionProcedure 68e3358487792e1f6c40accb55544a0f, server=e3d0adb87c01,40631,1731831247805 in 1.0810 sec 2024-11-17T08:17:21,046 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=174, resume processing ppid=173 2024-11-17T08:17:21,046 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, ppid=173, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=68e3358487792e1f6c40accb55544a0f, UNASSIGN in 1.0850 sec 2024-11-17T08:17:21,048 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-17T08:17:21,048 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.0870 sec 2024-11-17T08:17:21,049 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731831441048"}]},"ts":"1731831441048"} 2024-11-17T08:17:21,050 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T08:17:21,050 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T08:17:21,051 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.0940 sec 2024-11-17T08:17:21,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-17T08:17:21,093 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:17:21,094 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-17T08:17:21,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T08:17:21,095 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=176, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T08:17:21,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-17T08:17:21,096 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=176, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T08:17:21,098 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:21,100 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A, FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B, FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C, FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/recovered.edits] 2024-11-17T08:17:21,102 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/1c78d1aaae044e8bb18620d7929302d4 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/1c78d1aaae044e8bb18620d7929302d4 2024-11-17T08:17:21,102 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/8b74281bc21d404ba910900264e8181d to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/8b74281bc21d404ba910900264e8181d 2024-11-17T08:17:21,103 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/e76c5e58567b42958d0897dc2f331b9a to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/A/e76c5e58567b42958d0897dc2f331b9a 2024-11-17T08:17:21,105 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/3d77cdd2766f47dc9cece0a8ed934cb8 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/3d77cdd2766f47dc9cece0a8ed934cb8 2024-11-17T08:17:21,105 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/74ebeeb69e964ccba77a2c949093d658 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/74ebeeb69e964ccba77a2c949093d658 2024-11-17T08:17:21,106 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/9b695465d3f445c5b1300a5ec8811664 to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/B/9b695465d3f445c5b1300a5ec8811664 2024-11-17T08:17:21,107 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/2c1fa9f5e68f44729ecaef7bfa87d2cd to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/2c1fa9f5e68f44729ecaef7bfa87d2cd 2024-11-17T08:17:21,108 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/3e367b193a99459f8f377b7cf554941e to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/3e367b193a99459f8f377b7cf554941e 2024-11-17T08:17:21,109 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/e8090da770184351b00743fe0f23803c to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/C/e8090da770184351b00743fe0f23803c 2024-11-17T08:17:21,110 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/recovered.edits/325.seqid to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f/recovered.edits/325.seqid 2024-11-17T08:17:21,111 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/default/TestAcidGuarantees/68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:21,111 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T08:17:21,111 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T08:17:21,112 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-17T08:17:21,113 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111705f32d18c01c40e3bddd7b3d9e1fcd62_68e3358487792e1f6c40accb55544a0f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111705f32d18c01c40e3bddd7b3d9e1fcd62_68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:21,114 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170a73b43297eb459e97c79d4d749bf643_68e3358487792e1f6c40accb55544a0f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170a73b43297eb459e97c79d4d749bf643_68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:21,115 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411171d6dc8e1f8324ce7a9edbf1756f2926c_68e3358487792e1f6c40accb55544a0f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411171d6dc8e1f8324ce7a9edbf1756f2926c_68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:21,115 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411171ecbca7b1c604396a888eff77a674237_68e3358487792e1f6c40accb55544a0f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411171ecbca7b1c604396a888eff77a674237_68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:21,116 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111722e9f138fdcf4afb8c1c96e0613da950_68e3358487792e1f6c40accb55544a0f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111722e9f138fdcf4afb8c1c96e0613da950_68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:21,117 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172800dc683152490c8d6d51b973bb91db_68e3358487792e1f6c40accb55544a0f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411172800dc683152490c8d6d51b973bb91db_68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:21,117 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111730475c9f375041c7a91e32c51c8ce6a7_68e3358487792e1f6c40accb55544a0f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111730475c9f375041c7a91e32c51c8ce6a7_68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:21,118 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111766b36180abe64ed4896ed6b1a5f24a82_68e3358487792e1f6c40accb55544a0f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111766b36180abe64ed4896ed6b1a5f24a82_68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:21,119 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411177058e670f23c484e9fb6b8e520ef2c99_68e3358487792e1f6c40accb55544a0f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411177058e670f23c484e9fb6b8e520ef2c99_68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:21,119 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411179f0d86f0ccb14849a57d43cc2240df76_68e3358487792e1f6c40accb55544a0f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411179f0d86f0ccb14849a57d43cc2240df76_68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:21,120 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a72d0d4d266c4e5093731bf19c50335d_68e3358487792e1f6c40accb55544a0f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a72d0d4d266c4e5093731bf19c50335d_68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:21,121 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a78d1dd8cd3e4ba7aaaf25c445726916_68e3358487792e1f6c40accb55544a0f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a78d1dd8cd3e4ba7aaaf25c445726916_68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:21,121 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117aaf8595c7ccd4b2da55658a67436845b_68e3358487792e1f6c40accb55544a0f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117aaf8595c7ccd4b2da55658a67436845b_68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:21,122 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ae118cb64f09414bbe48b44fca0de871_68e3358487792e1f6c40accb55544a0f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ae118cb64f09414bbe48b44fca0de871_68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:21,123 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c704901978174e6ab5f0754180680713_68e3358487792e1f6c40accb55544a0f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c704901978174e6ab5f0754180680713_68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:21,123 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d1174a2690614965a674460d8ea82767_68e3358487792e1f6c40accb55544a0f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d1174a2690614965a674460d8ea82767_68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:21,124 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d42a628275f644159d613858e2d331a7_68e3358487792e1f6c40accb55544a0f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d42a628275f644159d613858e2d331a7_68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:21,125 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117dbe01fa0f2fe4b8db1d7613bc24febe7_68e3358487792e1f6c40accb55544a0f to hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117dbe01fa0f2fe4b8db1d7613bc24febe7_68e3358487792e1f6c40accb55544a0f 2024-11-17T08:17:21,125 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T08:17:21,126 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=176, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T08:17:21,128 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T08:17:21,130 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T08:17:21,130 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=176, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T08:17:21,130 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T08:17:21,131 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731831441130"}]},"ts":"9223372036854775807"} 2024-11-17T08:17:21,132 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T08:17:21,132 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 68e3358487792e1f6c40accb55544a0f, NAME => 'TestAcidGuarantees,,1731831408441.68e3358487792e1f6c40accb55544a0f.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T08:17:21,132 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T08:17:21,132 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731831441132"}]},"ts":"9223372036854775807"} 2024-11-17T08:17:21,133 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T08:17:21,134 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=176, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T08:17:21,135 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 40 msec 2024-11-17T08:17:21,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45089 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-17T08:17:21,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T08:17:21,204 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T08:17:21,212 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 250), OpenFileDescriptor=448 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=375 (was 355) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2192 (was 2919) 2024-11-17T08:17:21,212 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-17T08:17:21,212 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T08:17:21,213 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T08:17:21,213 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:17:21,213 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:17:21,213 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T08:17:21,213 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-17T08:17:21,213 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1258311727, stopped=false 2024-11-17T08:17:21,213 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=e3d0adb87c01,45089,1731831246985 2024-11-17T08:17:21,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45089-0x10147dba8ff0000, quorum=127.0.0.1:63364, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-17T08:17:21,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40631-0x10147dba8ff0001, quorum=127.0.0.1:63364, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-17T08:17:21,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45089-0x10147dba8ff0000, quorum=127.0.0.1:63364, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T08:17:21,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40631-0x10147dba8ff0001, quorum=127.0.0.1:63364, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T08:17:21,248 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T08:17:21,249 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T08:17:21,249 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T08:17:21,249 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:17:21,249 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e3d0adb87c01,40631,1731831247805' ***** 2024-11-17T08:17:21,249 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-17T08:17:21,250 INFO [RS:0;e3d0adb87c01:40631 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-17T08:17:21,250 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40631-0x10147dba8ff0001, quorum=127.0.0.1:63364, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T08:17:21,250 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45089-0x10147dba8ff0000, quorum=127.0.0.1:63364, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T08:17:21,250 INFO [RS:0;e3d0adb87c01:40631 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-17T08:17:21,250 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-17T08:17:21,250 INFO [RS:0;e3d0adb87c01:40631 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-17T08:17:21,250 INFO [RS:0;e3d0adb87c01:40631 {}] regionserver.HRegionServer(959): stopping server e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:21,250 INFO [RS:0;e3d0adb87c01:40631 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T08:17:21,250 INFO [RS:0;e3d0adb87c01:40631 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;e3d0adb87c01:40631. 2024-11-17T08:17:21,250 DEBUG [RS:0;e3d0adb87c01:40631 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T08:17:21,250 DEBUG [RS:0;e3d0adb87c01:40631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:17:21,250 INFO [RS:0;e3d0adb87c01:40631 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-17T08:17:21,250 INFO [RS:0;e3d0adb87c01:40631 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-17T08:17:21,250 INFO [RS:0;e3d0adb87c01:40631 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-17T08:17:21,251 INFO [RS:0;e3d0adb87c01:40631 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-17T08:17:21,251 INFO [RS:0;e3d0adb87c01:40631 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-17T08:17:21,251 DEBUG [RS:0;e3d0adb87c01:40631 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-17T08:17:21,251 DEBUG [RS_CLOSE_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-17T08:17:21,251 DEBUG [RS:0;e3d0adb87c01:40631 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T08:17:21,251 INFO [RS_CLOSE_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-17T08:17:21,251 DEBUG [RS_CLOSE_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-17T08:17:21,252 DEBUG [RS_CLOSE_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-17T08:17:21,252 DEBUG [RS_CLOSE_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-17T08:17:21,252 INFO [RS_CLOSE_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-17T08:17:21,274 DEBUG [RS_CLOSE_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/hbase/meta/1588230740/.tmp/info/cd7f2ec1d85447b783536dfe8d1dc8e5 is 95, key is TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7./info:/1731831287375/DeleteFamily/seqid=0 2024-11-17T08:17:21,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742450_1626 (size=6514) 2024-11-17T08:17:21,340 INFO [regionserver/e3d0adb87c01:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T08:17:21,452 DEBUG [RS:0;e3d0adb87c01:40631 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T08:17:21,652 DEBUG [RS:0;e3d0adb87c01:40631 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T08:17:21,677 INFO [RS_CLOSE_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/hbase/meta/1588230740/.tmp/info/cd7f2ec1d85447b783536dfe8d1dc8e5 2024-11-17T08:17:21,694 DEBUG [RS_CLOSE_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/hbase/meta/1588230740/.tmp/ns/130d148405734e18a405d8c103a6657a is 93, key is TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7./ns:/1731831287336/DeleteFamily/seqid=0 2024-11-17T08:17:21,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742451_1627 (size=5906) 2024-11-17T08:17:21,852 DEBUG [RS:0;e3d0adb87c01:40631 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T08:17:22,052 DEBUG [RS:0;e3d0adb87c01:40631 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T08:17:22,092 INFO [regionserver/e3d0adb87c01:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-17T08:17:22,092 INFO [regionserver/e3d0adb87c01:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-17T08:17:22,099 INFO [RS_CLOSE_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/hbase/meta/1588230740/.tmp/ns/130d148405734e18a405d8c103a6657a 2024-11-17T08:17:22,121 DEBUG [RS_CLOSE_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/hbase/meta/1588230740/.tmp/rep_barrier/45689ad868424323a77288470b70d20e is 102, key is TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7./rep_barrier:/1731831287336/DeleteFamily/seqid=0 2024-11-17T08:17:22,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742452_1628 (size=6025) 2024-11-17T08:17:22,252 DEBUG [RS:0;e3d0adb87c01:40631 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T08:17:22,453 DEBUG [RS:0;e3d0adb87c01:40631 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T08:17:22,525 INFO [RS_CLOSE_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/hbase/meta/1588230740/.tmp/rep_barrier/45689ad868424323a77288470b70d20e 2024-11-17T08:17:22,546 DEBUG [RS_CLOSE_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/hbase/meta/1588230740/.tmp/table/9df2a96aa4a545fe816c979d2e52a789 is 96, key is TestAcidGuarantees,,1731831252799.988129a9cf65d53c5913e9c54b149fd7./table:/1731831287336/DeleteFamily/seqid=0 2024-11-17T08:17:22,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742453_1629 (size=5936) 2024-11-17T08:17:22,653 DEBUG [RS:0;e3d0adb87c01:40631 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T08:17:22,853 DEBUG [RS:0;e3d0adb87c01:40631 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T08:17:22,950 INFO [RS_CLOSE_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/hbase/meta/1588230740/.tmp/table/9df2a96aa4a545fe816c979d2e52a789 2024-11-17T08:17:22,954 DEBUG [RS_CLOSE_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/hbase/meta/1588230740/.tmp/info/cd7f2ec1d85447b783536dfe8d1dc8e5 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/hbase/meta/1588230740/info/cd7f2ec1d85447b783536dfe8d1dc8e5 2024-11-17T08:17:22,957 INFO [RS_CLOSE_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/hbase/meta/1588230740/info/cd7f2ec1d85447b783536dfe8d1dc8e5, entries=12, sequenceid=90, filesize=6.4 K 2024-11-17T08:17:22,958 DEBUG [RS_CLOSE_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/hbase/meta/1588230740/.tmp/ns/130d148405734e18a405d8c103a6657a as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/hbase/meta/1588230740/ns/130d148405734e18a405d8c103a6657a 2024-11-17T08:17:22,961 INFO [RS_CLOSE_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/hbase/meta/1588230740/ns/130d148405734e18a405d8c103a6657a, entries=8, sequenceid=90, filesize=5.8 K 2024-11-17T08:17:22,962 DEBUG [RS_CLOSE_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/hbase/meta/1588230740/.tmp/rep_barrier/45689ad868424323a77288470b70d20e as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/hbase/meta/1588230740/rep_barrier/45689ad868424323a77288470b70d20e 2024-11-17T08:17:22,965 INFO [RS_CLOSE_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/hbase/meta/1588230740/rep_barrier/45689ad868424323a77288470b70d20e, entries=6, sequenceid=90, filesize=5.9 K 2024-11-17T08:17:22,965 DEBUG [RS_CLOSE_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/hbase/meta/1588230740/.tmp/table/9df2a96aa4a545fe816c979d2e52a789 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/hbase/meta/1588230740/table/9df2a96aa4a545fe816c979d2e52a789 2024-11-17T08:17:22,969 INFO [RS_CLOSE_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/hbase/meta/1588230740/table/9df2a96aa4a545fe816c979d2e52a789, entries=7, sequenceid=90, filesize=5.8 K 2024-11-17T08:17:22,970 INFO [RS_CLOSE_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1718ms, sequenceid=90, compaction requested=false 2024-11-17T08:17:22,974 DEBUG [RS_CLOSE_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-17T08:17:22,974 DEBUG [RS_CLOSE_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-17T08:17:22,974 INFO [RS_CLOSE_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-17T08:17:22,974 DEBUG [RS_CLOSE_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731831441251Running coprocessor pre-close hooks at 1731831441251Disabling compacts and flushes for region at 1731831441251Disabling writes for close at 1731831441252 (+1 ms)Obtaining lock to block concurrent updates at 1731831441252Preparing flush snapshotting stores in 1588230740 at 1731831441252Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731831441252Flushing stores of hbase:meta,,1.1588230740 at 1731831441253 (+1 ms)Flushing 1588230740/info: creating writer at 1731831441253Flushing 1588230740/info: appending metadata at 1731831441273 (+20 ms)Flushing 1588230740/info: closing flushed file at 1731831441273Flushing 1588230740/ns: creating writer at 1731831441681 (+408 ms)Flushing 1588230740/ns: appending metadata at 1731831441694 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1731831441694Flushing 1588230740/rep_barrier: creating writer at 1731831442102 (+408 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731831442120 (+18 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731831442120Flushing 1588230740/table: creating writer at 1731831442528 (+408 ms)Flushing 1588230740/table: appending metadata at 1731831442546 (+18 ms)Flushing 1588230740/table: closing flushed file at 1731831442546Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3893c4fd: reopening flushed file at 1731831442953 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31ccd6ab: reopening flushed file at 1731831442957 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e306f97: reopening flushed file at 1731831442961 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3eb25554: reopening flushed file at 1731831442965 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1718ms, sequenceid=90, compaction requested=false at 1731831442970 (+5 ms)Writing region close event to WAL at 1731831442971 (+1 ms)Running coprocessor post-close hooks at 1731831442974 (+3 ms)Closed at 1731831442974 2024-11-17T08:17:22,975 DEBUG [RS_CLOSE_META-regionserver/e3d0adb87c01:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-17T08:17:23,053 INFO [RS:0;e3d0adb87c01:40631 {}] regionserver.HRegionServer(976): stopping server e3d0adb87c01,40631,1731831247805; all regions closed. 2024-11-17T08:17:23,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741834_1010 (size=25344) 2024-11-17T08:17:23,060 DEBUG [RS:0;e3d0adb87c01:40631 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/oldWALs 2024-11-17T08:17:23,060 INFO [RS:0;e3d0adb87c01:40631 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e3d0adb87c01%2C40631%2C1731831247805.meta:.meta(num 1731831252285) 2024-11-17T08:17:23,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741832_1008 (size=17742264) 2024-11-17T08:17:23,063 DEBUG [RS:0;e3d0adb87c01:40631 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/oldWALs 2024-11-17T08:17:23,063 INFO [RS:0;e3d0adb87c01:40631 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e3d0adb87c01%2C40631%2C1731831247805:(num 1731831251209) 2024-11-17T08:17:23,063 DEBUG [RS:0;e3d0adb87c01:40631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T08:17:23,063 INFO [RS:0;e3d0adb87c01:40631 {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T08:17:23,063 INFO [RS:0;e3d0adb87c01:40631 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-17T08:17:23,064 INFO [RS:0;e3d0adb87c01:40631 {}] hbase.ChoreService(370): Chore service for: regionserver/e3d0adb87c01:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-17T08:17:23,064 INFO [RS:0;e3d0adb87c01:40631 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-17T08:17:23,064 INFO [regionserver/e3d0adb87c01:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-17T08:17:23,065 INFO [RS:0;e3d0adb87c01:40631 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40631 2024-11-17T08:17:23,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40631-0x10147dba8ff0001, quorum=127.0.0.1:63364, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e3d0adb87c01,40631,1731831247805 2024-11-17T08:17:23,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45089-0x10147dba8ff0000, quorum=127.0.0.1:63364, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-17T08:17:23,080 INFO [RS:0;e3d0adb87c01:40631 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T08:17:23,081 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e3d0adb87c01,40631,1731831247805] 2024-11-17T08:17:23,101 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e3d0adb87c01,40631,1731831247805 already deleted, retry=false 2024-11-17T08:17:23,101 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e3d0adb87c01,40631,1731831247805 expired; onlineServers=0 2024-11-17T08:17:23,101 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'e3d0adb87c01,45089,1731831246985' ***** 2024-11-17T08:17:23,101 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-17T08:17:23,101 INFO [M:0;e3d0adb87c01:45089 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T08:17:23,101 INFO [M:0;e3d0adb87c01:45089 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-17T08:17:23,102 DEBUG [M:0;e3d0adb87c01:45089 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-17T08:17:23,102 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-17T08:17:23,102 DEBUG [M:0;e3d0adb87c01:45089 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-17T08:17:23,102 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster-HFileCleaner.large.0-1731831250915 {}] cleaner.HFileCleaner(306): Exit Thread[master/e3d0adb87c01:0:becomeActiveMaster-HFileCleaner.large.0-1731831250915,5,FailOnTimeoutGroup] 2024-11-17T08:17:23,102 DEBUG [master/e3d0adb87c01:0:becomeActiveMaster-HFileCleaner.small.0-1731831250915 {}] cleaner.HFileCleaner(306): Exit Thread[master/e3d0adb87c01:0:becomeActiveMaster-HFileCleaner.small.0-1731831250915,5,FailOnTimeoutGroup] 2024-11-17T08:17:23,102 INFO [M:0;e3d0adb87c01:45089 {}] hbase.ChoreService(370): Chore service for: master/e3d0adb87c01:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-17T08:17:23,102 INFO [M:0;e3d0adb87c01:45089 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-17T08:17:23,102 DEBUG [M:0;e3d0adb87c01:45089 {}] master.HMaster(1795): Stopping service threads 2024-11-17T08:17:23,102 INFO [M:0;e3d0adb87c01:45089 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-17T08:17:23,102 INFO [M:0;e3d0adb87c01:45089 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T08:17:23,103 ERROR [M:0;e3d0adb87c01:45089 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:37737 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:37737,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-17T08:17:23,103 INFO [M:0;e3d0adb87c01:45089 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-17T08:17:23,103 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-17T08:17:23,111 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45089-0x10147dba8ff0000, quorum=127.0.0.1:63364, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-17T08:17:23,111 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45089-0x10147dba8ff0000, quorum=127.0.0.1:63364, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T08:17:23,112 DEBUG [M:0;e3d0adb87c01:45089 {}] zookeeper.ZKUtil(347): master:45089-0x10147dba8ff0000, quorum=127.0.0.1:63364, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-17T08:17:23,112 WARN [M:0;e3d0adb87c01:45089 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-17T08:17:23,112 INFO [M:0;e3d0adb87c01:45089 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/.lastflushedseqids 2024-11-17T08:17:23,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742454_1630 (size=72) 2024-11-17T08:17:23,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40631-0x10147dba8ff0001, quorum=127.0.0.1:63364, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T08:17:23,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40631-0x10147dba8ff0001, quorum=127.0.0.1:63364, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T08:17:23,191 INFO [RS:0;e3d0adb87c01:40631 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T08:17:23,192 INFO [RS:0;e3d0adb87c01:40631 {}] regionserver.HRegionServer(1031): Exiting; stopping=e3d0adb87c01,40631,1731831247805; zookeeper connection closed. 2024-11-17T08:17:23,192 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@21235a46 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@21235a46 2024-11-17T08:17:23,192 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-17T08:17:23,521 INFO [M:0;e3d0adb87c01:45089 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-17T08:17:23,521 INFO [M:0;e3d0adb87c01:45089 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-17T08:17:23,521 DEBUG [M:0;e3d0adb87c01:45089 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-17T08:17:23,521 INFO [M:0;e3d0adb87c01:45089 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T08:17:23,521 DEBUG [M:0;e3d0adb87c01:45089 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T08:17:23,521 DEBUG [M:0;e3d0adb87c01:45089 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-17T08:17:23,521 DEBUG [M:0;e3d0adb87c01:45089 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T08:17:23,521 INFO [M:0;e3d0adb87c01:45089 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=770.69 KB heapSize=943.77 KB 2024-11-17T08:17:23,535 DEBUG [M:0;e3d0adb87c01:45089 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/00d3cc7728364082a5f0e56fa640b76e is 82, key is hbase:meta,,1/info:regioninfo/1731831252367/Put/seqid=0 2024-11-17T08:17:23,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742455_1631 (size=5672) 2024-11-17T08:17:23,939 INFO [M:0;e3d0adb87c01:45089 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2130 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/00d3cc7728364082a5f0e56fa640b76e 2024-11-17T08:17:23,965 DEBUG [M:0;e3d0adb87c01:45089 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f25857e8626444c2a0b68e45e5fa58f8 is 2479, key is \x00\x00\x00\x00\x00\x00\x00\x95/proc:d/1731831411527/Put/seqid=0 2024-11-17T08:17:23,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742456_1632 (size=46389) 2024-11-17T08:17:24,368 INFO [M:0;e3d0adb87c01:45089 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770.13 KB at sequenceid=2130 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f25857e8626444c2a0b68e45e5fa58f8 2024-11-17T08:17:24,371 INFO [M:0;e3d0adb87c01:45089 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f25857e8626444c2a0b68e45e5fa58f8 2024-11-17T08:17:24,384 DEBUG [M:0;e3d0adb87c01:45089 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9715ecd4f4cf4d4f81bc783b004f37c7 is 69, key is e3d0adb87c01,40631,1731831247805/rs:state/1731831250960/Put/seqid=0 2024-11-17T08:17:24,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073742457_1633 (size=5156) 2024-11-17T08:17:24,787 INFO [M:0;e3d0adb87c01:45089 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2130 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9715ecd4f4cf4d4f81bc783b004f37c7 2024-11-17T08:17:24,791 DEBUG [M:0;e3d0adb87c01:45089 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/00d3cc7728364082a5f0e56fa640b76e as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/00d3cc7728364082a5f0e56fa640b76e 2024-11-17T08:17:24,793 INFO [M:0;e3d0adb87c01:45089 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/00d3cc7728364082a5f0e56fa640b76e, entries=8, sequenceid=2130, filesize=5.5 K 2024-11-17T08:17:24,794 DEBUG [M:0;e3d0adb87c01:45089 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f25857e8626444c2a0b68e45e5fa58f8 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f25857e8626444c2a0b68e45e5fa58f8 2024-11-17T08:17:24,796 INFO [M:0;e3d0adb87c01:45089 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f25857e8626444c2a0b68e45e5fa58f8 2024-11-17T08:17:24,796 INFO [M:0;e3d0adb87c01:45089 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f25857e8626444c2a0b68e45e5fa58f8, entries=176, sequenceid=2130, filesize=45.3 K 2024-11-17T08:17:24,797 DEBUG [M:0;e3d0adb87c01:45089 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9715ecd4f4cf4d4f81bc783b004f37c7 as hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9715ecd4f4cf4d4f81bc783b004f37c7 2024-11-17T08:17:24,800 INFO [M:0;e3d0adb87c01:45089 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/0d3a7712-f8c6-cf5f-2c1c-6dccc46d6e85/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9715ecd4f4cf4d4f81bc783b004f37c7, entries=1, sequenceid=2130, filesize=5.0 K 2024-11-17T08:17:24,800 INFO [M:0;e3d0adb87c01:45089 {}] regionserver.HRegion(3140): Finished flush of dataSize ~770.69 KB/789185, heapSize ~943.47 KB/966112, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1279ms, sequenceid=2130, compaction requested=false 2024-11-17T08:17:24,801 INFO [M:0;e3d0adb87c01:45089 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T08:17:24,801 DEBUG [M:0;e3d0adb87c01:45089 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731831443521Disabling compacts and flushes for region at 1731831443521Disabling writes for close at 1731831443521Obtaining lock to block concurrent updates at 1731831443521Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731831443521Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=789185, getHeapSize=966352, getOffHeapSize=0, getCellsCount=2331 at 1731831443522 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731831443522Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731831443522Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731831443535 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731831443535Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731831443942 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731831443964 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731831443964Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731831444372 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731831444384 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731831444384Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@552bbffe: reopening flushed file at 1731831444790 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1dba8782: reopening flushed file at 1731831444793 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@20e9eea6: reopening flushed file at 1731831444796 (+3 ms)Finished flush of dataSize ~770.69 KB/789185, heapSize ~943.47 KB/966112, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1279ms, sequenceid=2130, compaction requested=false at 1731831444800 (+4 ms)Writing region close event to WAL at 1731831444801 (+1 ms)Closed at 1731831444801 2024-11-17T08:17:24,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40497 is added to blk_1073741830_1006 (size=928858) 2024-11-17T08:17:24,804 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-17T08:17:24,804 INFO [M:0;e3d0adb87c01:45089 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-17T08:17:24,804 INFO [M:0;e3d0adb87c01:45089 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:45089 2024-11-17T08:17:24,805 INFO [M:0;e3d0adb87c01:45089 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T08:17:24,948 INFO [M:0;e3d0adb87c01:45089 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T08:17:24,948 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45089-0x10147dba8ff0000, quorum=127.0.0.1:63364, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T08:17:24,948 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45089-0x10147dba8ff0000, quorum=127.0.0.1:63364, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T08:17:24,953 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-17T08:17:24,955 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-17T08:17:24,955 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-17T08:17:24,955 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-17T08:17:24,955 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5720963-478b-ee25-58c5-86f892de1407/hadoop.log.dir/,STOPPED} 2024-11-17T08:17:24,958 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-17T08:17:24,958 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-17T08:17:24,958 WARN [BP-2116440293-172.17.0.3-1731831243397 heartbeating to localhost/127.0.0.1:37737 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-17T08:17:24,958 WARN [BP-2116440293-172.17.0.3-1731831243397 heartbeating to localhost/127.0.0.1:37737 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2116440293-172.17.0.3-1731831243397 (Datanode Uuid bc39c235-fe1b-4242-b2b6-1d3bc0ce3f3c) service to localhost/127.0.0.1:37737 2024-11-17T08:17:24,961 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5720963-478b-ee25-58c5-86f892de1407/cluster_aa2b9c1b-2c01-fdb4-e68c-d41e93bcbc98/data/data1/current/BP-2116440293-172.17.0.3-1731831243397 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-17T08:17:24,961 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5720963-478b-ee25-58c5-86f892de1407/cluster_aa2b9c1b-2c01-fdb4-e68c-d41e93bcbc98/data/data2/current/BP-2116440293-172.17.0.3-1731831243397 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-17T08:17:24,962 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-17T08:17:24,969 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-17T08:17:24,970 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-17T08:17:24,970 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-17T08:17:24,970 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-17T08:17:24,970 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e5720963-478b-ee25-58c5-86f892de1407/hadoop.log.dir/,STOPPED} 2024-11-17T08:17:24,985 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-17T08:17:25,112 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down